Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 119 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ namespace ggml::hrx {
namespace {

static constexpr KernelCatalogRef kGetRowsF32Kernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32");
static constexpr KernelCatalogRef kGetRowsScalar3dF32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scalar_3d_f32");
static constexpr KernelCatalogRef kRepeatScalar3dF32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_repeat_scalar_3d_f32");
static constexpr KernelCatalogRef kGetRowsF32NextKernel = GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_f32_next");
static constexpr KernelCatalogRef kGetRowsScaleF32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_get_rows_scale_f32");
Expand Down Expand Up @@ -153,6 +157,105 @@ static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });
}

static bool match_repeat_scalar_3d_f32_input(const Graph & graph, const GraphNode * node,
const Value *& source, const Value *& repeated) {
const bool is_repeat = node != nullptr && node->op == GGML_OP_REPEAT;
const bool has_one_input = node != nullptr && node->inputs.size() == 1;
if (!is_repeat || !has_one_input) {
return false;
}

source = graph_value(graph, node->inputs[0]);
repeated = graph_value(graph, node->output);
const bool has_values = source != nullptr && repeated != nullptr;
if (!has_values) {
return false;
}

const bool has_types = source->type == GGML_TYPE_F32 && repeated->type == GGML_TYPE_F32;
const bool has_source_shape = source->ne[0] == 1 && source->ne[1] == 128 &&
source->ne[2] == 1 && source->ne[3] == 1;
const bool has_repeat_shape = repeated->ne[0] == 1 && repeated->ne[1] == 128 &&
repeated->ne[3] == 1 && is_supported_token_count(repeated->ne[2]);
const bool has_layout = source->contiguous && repeated->contiguous;
return has_types && has_source_shape && has_repeat_shape && has_layout;
}

static bool match_repeat_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) {
const Value * source = nullptr;
const Value * repeated = nullptr;
if (!match_repeat_scalar_3d_f32_input(context.graph, context.root_node, source, repeated)) {
return false;
}

Dispatch dispatch;
dispatch.kernel = make_kernel_specialization(kRepeatScalar3dF32Kernel);
dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic;
dispatch.kernel.integer_parameters.emplace("token_count", repeated->ne[2]);
dispatch.bindings.push_back({ source->id, 0, source->byte_count });
dispatch.bindings.push_back({ repeated->id, 0, repeated->byte_count });

match.covered_nodes.push_back(context.root_index);
match.dispatches.push_back(std::move(dispatch));
return true;
}

static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & match) {
const GraphNode * node = context.root_node;
const bool is_get_rows = node != nullptr && node->op == GGML_OP_GET_ROWS;
const bool has_two_inputs = node != nullptr && node->inputs.size() == 2;
if (!is_get_rows || !has_two_inputs) {
return false;
}

const Value * scales = graph_value(context.graph, node->inputs[0]);
const Value * ids = graph_value(context.graph, node->inputs[1]);
const Value * output = graph_value(context.graph, node->output);
const bool has_values = scales != nullptr && ids != nullptr && output != nullptr;
if (!has_values) {
return false;
}

const int64_t token_count = output->ne[2];
const bool has_supported_tokens = is_supported_token_count(token_count);
const bool has_scale_shape = scales->ne[0] == 1 && scales->ne[1] == 128 &&
scales->ne[2] == token_count && scales->ne[3] == 1;
const bool has_id_shape = ids->ne[0] == 8 && ids->ne[1] == token_count &&
ids->ne[2] == 1 && ids->ne[3] == 1;
const bool has_output_shape = output->ne[0] == 1 && output->ne[1] == 8 &&
output->ne[3] == 1;
const bool has_types = scales->type == GGML_TYPE_F32 && ids->type == GGML_TYPE_I32 &&
output->type == GGML_TYPE_F32;
const bool has_contiguous_values = scales->contiguous && output->contiguous;
const bool has_id_columns = ids->nb[0] == sizeof(int32_t) && ids->nb[1] % sizeof(int32_t) == 0;
const bool has_supported_shape = has_supported_tokens && has_scale_shape && has_id_shape &&
has_output_shape && has_types && has_contiguous_values && has_id_columns;
if (!has_supported_shape) {
return false;
}

const size_t route_stride = ids->nb[1] / sizeof(int32_t);
const bool has_supported_stride = route_stride >= 8 && route_stride <= 512;
const size_t expected_id_bytes = (static_cast<size_t>(token_count) - 1) * ids->nb[1] + 8 * sizeof(int32_t);
const bool has_exact_id_span = ids->byte_count == expected_id_bytes;
if (!has_supported_stride || !has_exact_id_span) {
return false;
}

Dispatch dispatch;
dispatch.kernel = make_kernel_specialization(kGetRowsScalar3dF32Kernel);
dispatch.kernel.workload_specialization = WorkloadSpecialization::Dynamic;
dispatch.kernel.integer_parameters.emplace("token_count", token_count);
dispatch.kernel.integer_parameters.emplace("route_stride", static_cast<int64_t>(route_stride));
dispatch.bindings.push_back({ ids->id, 0, ids->byte_count });
dispatch.bindings.push_back({ scales->id, 0, scales->byte_count });
dispatch.bindings.push_back({ output->id, 0, output->byte_count });

match.covered_nodes.push_back(context.root_index);
match.dispatches.push_back(std::move(dispatch));
return true;
}

static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * node) {
GetRowsMatch match;
if (node == nullptr || node->op != GGML_OP_GET_ROWS || node->inputs.size() != 2) {
Expand Down Expand Up @@ -472,6 +575,22 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext &
} // namespace

void register_get_rows_dispatches(DispatchRegistryBuilder & registry) {
registry.add({
"common.repeat.scalar_3d_f32",
GGML_OP_REPEAT,
DispatchMatchKind::SingleOp,
125,
DispatchSource::Common,
match_repeat_scalar_3d_f32_dispatch,
});
registry.add({
"common.get_rows.scalar_3d_f32",
GGML_OP_GET_ROWS,
DispatchMatchKind::SingleOp,
125,
DispatchSource::Common,
match_get_rows_scalar_3d_f32_dispatch,
});
registry.add({
"common.get_rows_scale.f32",
GGML_OP_GET_ROWS,
Expand Down
181 changes: 181 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-id.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ static constexpr KernelCatalogRef kMulMatIdTiledF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_publish_f32");
static constexpr KernelCatalogRef kMulMatIdSkinnyF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_skinny_input_f32_publish_f32");
static constexpr KernelCatalogRef kMulMatIdTiledScaledF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_scaled_publish_f32");
static constexpr KernelCatalogRef kMulMatIdSkinnyScaledF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_skinny_input_f32_scaled_publish_f32");
static constexpr KernelCatalogRef kMulMatIdTiledPostOpsF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_id_tiled_input_f32_postops_publish_f32");
static constexpr KernelCatalogRef kMulMatIdSkinnyPostOpsF32F32Kernel =
Expand Down Expand Up @@ -62,6 +66,21 @@ struct MulMatIdPostOpsMatch {
}
};

struct MulMatIdExpertScaleMatch {
CommonMulMatIdMatch root;
const Value * scale = nullptr;
const Value * output = nullptr;
const GraphNode * repeat = nullptr;
const GraphNode * gather = nullptr;
const GraphNode * multiply = nullptr;
int64_t route_stride = 0;

bool matched() const {
return root.matched() && scale != nullptr && output != nullptr && repeat != nullptr && gather != nullptr &&
multiply != nullptr;
}
};

static bool is_bias_shape(const Value & value, int64_t output_size) {
if (value.type != GGML_TYPE_F32 || !value.contiguous || value.ne[0] != output_size) {
return false;
Expand Down Expand Up @@ -236,6 +255,160 @@ static bool match_mul_mat_id_dispatch(const DispatchMatchContext & context, Disp
return true;
}

static MulMatIdExpertScaleMatch match_mul_mat_id_expert_scale(const DispatchMatchContext & context) {
MulMatIdExpertScaleMatch match;
const CommonMulMatIdMatch root =
common_match_mul_mat_id_any_format(context.graph, context.root_node, context.plan);
if (!root.matched()) {
return match;
}

const bool has_gemma_experts = root.expert_count == 128;
const bool has_eight_routes = root.route_count == 8;
const bool has_gemma_routing = has_gemma_experts && has_eight_routes;
const bool has_supported_format = root.weight_format == CommonMulMatWeightFormat::Q4K ||
root.weight_format == CommonMulMatWeightFormat::Q6K ||
root.weight_format == CommonMulMatWeightFormat::Q5_1 ||
root.weight_format == CommonMulMatWeightFormat::Q8_0;
const bool has_transient_matmul_output = root.output->kind == ValueKind::Transient;
const bool has_eligible_root = has_gemma_routing && has_supported_format && has_transient_matmul_output;
if (!has_eligible_root) {
return {};
}

const GraphNode * multiply =
common_mul_mat_id_find_only_consumer_with_op(context.graph, root.output->id, GGML_OP_MUL);
const bool has_multiply = multiply != nullptr && common_mul_mat_id_binary_node_is_mul(*multiply) &&
multiply->inputs[0] == root.output->id;
if (!has_multiply) {
return {};
}

const Value * selected = common_mul_mat_id_graph_value(context.graph, multiply->inputs[1]);
const Value * output = common_mul_mat_id_graph_value(context.graph, multiply->output);
const bool has_values = selected != nullptr && output != nullptr;
if (!has_values) {
return {};
}
const GraphNode * gather = context.graph.index().producer(selected->id);
const bool has_gather = gather != nullptr && gather->op == GGML_OP_GET_ROWS && gather->inputs.size() == 2;
const bool has_selected_shape = selected->type == GGML_TYPE_F32 &&
common_mul_mat_id_is_shape(*selected, 1, 8, root.token_count, 1);
const bool has_output_shape = output->type == GGML_TYPE_F32 && output->contiguous &&
common_mul_mat_id_same_shape(*output, *root.output);
const bool has_selected_consumer =
context.graph.index().consumers(selected->id).size() == 1 &&
context.graph.index().consumers(selected->id).front() == multiply;
const bool has_scale_gather = has_gather && has_selected_shape && has_output_shape && has_selected_consumer;
if (!has_scale_gather) {
return {};
}
const bool uses_same_ids = gather->inputs[1] == root.route_ids->id;
if (!uses_same_ids) {
return {};
}

const Value * repeated = common_mul_mat_id_graph_value(context.graph, gather->inputs[0]);
const GraphNode * repeat = repeated == nullptr ? nullptr : context.graph.index().producer(repeated->id);
const bool has_repeat = repeat != nullptr && repeat->op == GGML_OP_REPEAT && repeat->inputs.size() == 1;
const bool has_repeat_shape = repeated != nullptr && repeated->type == GGML_TYPE_F32 &&
common_mul_mat_id_is_shape(*repeated, 1, 128, root.token_count, 1) &&
repeated->kind == ValueKind::Transient;
const bool has_repeat_consumer = repeated != nullptr &&
context.graph.index().consumers(repeated->id).size() == 1 &&
context.graph.index().consumers(repeated->id).front() == gather;
const bool has_scale_repeat = has_repeat && has_repeat_shape && has_repeat_consumer;
if (!has_scale_repeat) {
return {};
}

const Value * scale = common_mul_mat_id_graph_value(context.graph, repeat->inputs[0]);
const bool has_scale_shape = scale != nullptr && scale->type == GGML_TYPE_F32 && scale->contiguous &&
common_mul_mat_id_is_shape(*scale, 1, 128, 1, 1) &&
scale->byte_count == 128 * sizeof(float);
const bool has_contiguous_repeat = repeated->contiguous;
const bool has_scale_source = has_scale_shape && has_contiguous_repeat;
if (!has_scale_source) {
return {};
}

const bool has_id_columns = root.route_ids->nb[0] == sizeof(int32_t) &&
root.route_ids->nb[1] % sizeof(int32_t) == 0;
if (!has_id_columns) {
return {};
}
const int64_t route_stride = static_cast<int64_t>(root.route_ids->nb[1] / sizeof(int32_t));
const bool has_supported_stride = route_stride >= 8 && route_stride <= 512;
const size_t expected_id_bytes = (static_cast<size_t>(root.token_count) - 1) * root.route_ids->nb[1] +
8 * sizeof(int32_t);
const bool has_exact_id_span = root.route_ids->byte_count == expected_id_bytes;
const bool has_supported_ids = has_supported_stride && has_exact_id_span;
if (!has_supported_ids) {
return {};
}

match.root = root;
match.scale = scale;
match.output = output;
match.repeat = repeat;
match.gather = gather;
match.multiply = multiply;
match.route_stride = route_stride;
return match;
}

static bool match_mul_mat_id_expert_scale_dispatch(const DispatchMatchContext & context,
DispatchMatch & dispatch_match) {
const MulMatIdExpertScaleMatch match = match_mul_mat_id_expert_scale(context);
if (!match.matched()) {
return false;
}

CommandPlanMoeRoutingBundle routing_bundle;
if (!common_mul_mat_id_ensure_moe_routing_bundle(context, match.root, dispatch_match, routing_bundle)) {
return false;
}

Dispatch dispatch;
const bool skinny_route = common_mul_mat_id_is_skinny_route(match.root);
dispatch.kernel = make_kernel_specialization(skinny_route ? kMulMatIdSkinnyScaledF32F32Kernel :
kMulMatIdTiledScaledF32F32Kernel);
dispatch.kernel.integer_parameters.emplace("token_count", match.root.token_count);
dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity",
common_mul_mat_id_to_config_value(match.root.token_count));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.input_size",
common_mul_mat_id_to_config_value(match.root.input_size));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.output_size",
common_mul_mat_id_to_config_value(match.root.output_size));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.expert_count",
common_mul_mat_id_to_config_value(match.root.expert_count));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.route_count",
common_mul_mat_id_to_config_value(match.root.route_count));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.input_route_count",
common_mul_mat_id_to_config_value(match.root.input_route_count));
dispatch.kernel.compile_parameters.emplace(
"ggml.mul_mat_id.weight_format",
common_mul_mat_id_to_config_value(common_mul_mat_format_config_value(match.root.weight_format)));
dispatch.kernel.compile_parameters.emplace("ggml.mul_mat_id.scale.route_stride",
common_mul_mat_id_to_config_value(match.route_stride));
dispatch.bindings.push_back({ match.root.input->id, 0, match.root.input->byte_count });
dispatch.bindings.push_back({ routing_bundle.expert_table, 0, routing_bundle.expert_table_byte_count });
dispatch.bindings.push_back({ routing_bundle.partition_table, 0, routing_bundle.partition_table_byte_count });
dispatch.bindings.push_back({ match.root.weight->id, 0, match.root.weight->byte_count });
dispatch.bindings.push_back({ match.scale->id, 0, match.scale->byte_count });
dispatch.bindings.push_back({ match.root.route_ids->id, 0, match.root.route_ids->byte_count });
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });

for (const GraphNode * covered_node : { context.root_node, match.repeat, match.gather, match.multiply }) {
if (!append_covered_node_index_once(context.graph, context.covered_nodes, covered_node,
dispatch_match.covered_nodes)) {
return false;
}
}
dispatch_match.dispatches.push_back(std::move(dispatch));
return true;
}

static bool match_mul_mat_id_postops_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) {
const MulMatIdPostOpsMatch match = match_mul_mat_id_postops(context);
if (!match.matched()) {
Expand Down Expand Up @@ -331,6 +504,14 @@ static bool match_mul_mat_id_postops_dispatch(const DispatchMatchContext & conte
}

void register_mul_mat_id_dispatches(DispatchRegistryBuilder & registry) {
registry.add({
"common.mul_mat_id.expert_scale.f32_f32_wmma",
GGML_OP_MUL_MAT_ID,
DispatchMatchKind::Fused,
190,
DispatchSource::Common,
match_mul_mat_id_expert_scale_dispatch,
});
registry.add({
"common.mul_mat_id_postops.f32_f32_wmma",
GGML_OP_MUL_MAT_ID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -536,8 +536,7 @@ static bool match_moe_router_projection_top8_fused_decode_dispatch(const Dispatc
dispatch.kernel.compile_parameters.emplace("qwen3_moe.workload.token_capacity",
to_config_value(match.top8.token_count));

const size_t route_id_length =
static_cast<size_t>(match.top8.token_count * match.top8.route_stride) * sizeof(int32_t);
const size_t route_id_length = match.top8.route_ids->byte_count;
dispatch.bindings.push_back({ match.input->id, 0, match.input->byte_count });
dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count });
dispatch.bindings.push_back({ match.logits->id, 0, match.logits->byte_count });
Expand Down Expand Up @@ -576,8 +575,7 @@ static bool match_moe_router_top8_dispatch(const DispatchMatchContext & context,
dispatch.kernel.compile_parameters.emplace("qwen3_moe.workload.token_capacity",
to_config_value(router_match.token_count));

const size_t route_id_length =
static_cast<size_t>(router_match.token_count * router_match.route_stride) * sizeof(int32_t);
const size_t route_id_length = router_match.route_ids->byte_count;
dispatch.bindings.push_back({ router_match.logits->id, 0, router_match.logits->byte_count });
dispatch.bindings.push_back({ router_match.route_ids->id, 0, route_id_length });
dispatch.bindings.push_back({ router_match.route_weights->id, 0, router_match.route_weights->byte_count });
Expand Down
Loading
Loading