Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions ggml/src/ggml-hrx/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ ggml_add_backend_library(ggml-hrx
dispatch_registration/common/dispatch-mul-mat-common.h
dispatch_registration/common/dispatch-mul-mat-weight-format.h
dispatch_registration/common/dispatch-mul-mat.h
dispatch_registration/common/dispatch-amax-scale.cpp
dispatch_registration/common/dispatch-amax-scale.h
dispatch_registration/common/dispatch-rope-set-rows.cpp
dispatch_registration/common/dispatch-rope-set-rows.h
dispatch_registration/common/dispatch-rope-utils.h
Expand Down
189 changes: 189 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-amax-scale.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,189 @@
#include "dispatch-amax-scale.h"

#include "ggml.h"
#include "graph/graph-matcher.h"
#include "kernel-corpus/kernel-corpus-catalog-verify.h"

#include <cstdint>
#include <utility>

namespace ggml::hrx {
namespace {

static constexpr KernelCatalogRef kAmaxScaleF32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_amax_scale_f32");

static bool matches_unary(const GraphNode & node, UnaryKind kind) {
const UnaryParams * params = op_params_as<UnaryParams>(node.params);
if (params == nullptr) {
return false;
}
return params->op == kind;
}

static bool matches_scale(const GraphNode & node, float factor) {
const ScaleParams * params = op_params_as<ScaleParams>(node.params);
if (params == nullptr) {
return false;
}
const bool same_factor = params->scale == factor;
const bool no_bias = params->bias == 0.0f;
return same_factor && no_bias;
}

static bool match_amax_scale_f32(const DispatchMatchContext & context, DispatchMatch & match) {
const GraphNode * node = context.root_node;
if (node == nullptr) {
return false;
}
const Graph & graph = context.graph;
// Intermediate values must stay private: this dispatch publishes only x/s and s.
const ggml_op ops[] = { GGML_OP_UNARY, GGML_OP_POOL_1D, GGML_OP_SCALE, GGML_OP_LOG,
GGML_OP_SCALE, GGML_OP_UNARY, GGML_OP_CLAMP, GGML_OP_SCALE, GGML_OP_UNARY };
std::vector<const GraphNode *> chain;
for (size_t i = 0; i < 9; ++i) {
const bool right_op = node->op == ops[i];
const bool one_input = node->inputs.size() == 1;
const bool expected_node = right_op && one_input;
if (!expected_node) {
return false;
}
const Value * value = graph.values().find(node->output);
if (value == nullptr) {
return false;
}
const bool f32 = value->type == GGML_TYPE_F32;
const bool contiguous = value->contiguous;
const bool supported_value = f32 && contiguous;
if (!supported_value) {
return false;
}
chain.push_back(node);
if (i == 8) {
break;
}
const auto & consumers = graph.index().consumers(node->output);
const bool private_value = value->kind == ValueKind::Transient;
const bool one_consumer = consumers.size() == 1;
const bool can_elide = private_value && one_consumer;
if (!can_elide) {
return false;
}
node = consumers.front();
}
const bool abs_input = matches_unary(*chain[0], UnaryKind::Abs);
const bool scale_amax = matches_scale(*chain[2], 1.0f / 1024.0f);
const bool logarithm = matches_unary(*chain[3], UnaryKind::Log);
const bool scale_log = matches_scale(*chain[4], 1.4426950408889634f);
const bool ceil_log = matches_unary(*chain[5], UnaryKind::Ceil);
const bool scale_exp = matches_scale(*chain[7], 0.6931471805599453f);
const bool exponential = matches_unary(*chain[8], UnaryKind::Exp);
const bool exact_math = abs_input && scale_amax && logarithm && scale_log && ceil_log && scale_exp && exponential;
if (!exact_math) {
return false;
}
const Pool1DParams * pool = op_params_as<Pool1DParams>(chain[1]->params);
const ClampParams * clamp = op_params_as<ClampParams>(chain[6]->params);
const Value * input = graph.values().find(chain[0]->inputs[0]);
const bool has_pool = pool != nullptr;
const bool has_clamp = clamp != nullptr;
const bool has_input = input != nullptr;
const bool all_present = has_pool && has_clamp && has_input;
if (!all_present) {
return false;
}
const bool max_pool = pool->op == GGML_OP_POOL_MAX;
const bool full_width = pool->kernel == input->ne[0];
const bool full_stride = pool->stride == pool->kernel;
const bool no_padding = pool->padding == 0;
const bool clamp_min = clamp->min == 0.0f;
const bool clamp_max = clamp->max == 127.0f;
const bool input_f32 = input->type == GGML_TYPE_F32;
const bool bounded_width = input->ne[0] > 0 && input->ne[0] <= 32768;
const bool bounded_elements = input->element_count > 0 && input->element_count <= 134217728;
const bool supported = max_pool && full_width && full_stride && no_padding && clamp_min && clamp_max &&
input_f32 && input->contiguous && bounded_width && bounded_elements;
if (!supported) {
return false;
}
const Value * scale = graph.values().find(chain.back()->output);
const int64_t rows = input->element_count / input->ne[0];
const bool bounded_rows = rows <= 1048576;
if (!bounded_rows) {
return false;
}
for (size_t i = 0; i < chain.size(); ++i) {
const Value * value = graph.values().find(chain[i]->output);
auto expected_shape = input->ne;
if (i > 0) {
expected_shape[0] = 1;
}
const bool same_shape = value->ne == expected_shape;
const bool separate_input = value->storage_root != input->storage_root;
const bool valid_output = same_shape && separate_input;
if (!valid_output) {
return false;
}
}
const GraphNode * division = nullptr;
for (const GraphNode * consumer : graph.index().consumers(scale->id)) {
const bool is_div = consumer->op == GGML_OP_DIV;
const bool two_inputs = consumer->inputs.size() == 2;
const bool candidate = is_div && two_inputs;
if (!candidate) {
continue;
}
const bool numerator = consumer->inputs[0] == input->id;
const bool denominator = consumer->inputs[1] == scale->id;
const bool exact_division = numerator && denominator;
if (exact_division) {
division = consumer;
break;
}
}
if (division == nullptr) {
return false;
}
const Value * output = graph.values().find(division->output);
if (output == nullptr) {
return false;
}
const bool output_f32 = output->type == GGML_TYPE_F32;
const bool same_shape = output->ne == input->ne;
const bool separate_input = output->storage_root != input->storage_root;
const bool separate_scale = output->storage_root != scale->storage_root;
const bool valid_output = output_f32 && output->contiguous && same_shape && separate_input && separate_scale;
if (!valid_output) {
return false;
}
chain.push_back(division);
for (const GraphNode * covered : chain) {
if (!append_covered_node_index_once(graph, context.covered_nodes, covered, match.covered_nodes)) {
return false;
}
}
Dispatch dispatch;
dispatch.kernel = make_kernel_specialization(kAmaxScaleF32Kernel);
dispatch.kernel.integer_parameters.emplace("row_count", rows);
dispatch.kernel.integer_parameters.emplace("row_width", input->ne[0]);
dispatch.bindings.push_back({ input->id, 0, input->byte_count });
dispatch.bindings.push_back({ output->id, 0, output->byte_count });
dispatch.bindings.push_back({ scale->id, 0, scale->byte_count });
match.dispatches.push_back(std::move(dispatch));
return true;
}

} // namespace

void register_amax_scale_dispatch(DispatchRegistryBuilder & registry) {
registry.add({
"common.amax_scale_f32",
GGML_OP_UNARY,
DispatchMatchKind::Fused,
400,
DispatchSource::Common,
match_amax_scale_f32,
});
}

} // namespace ggml::hrx
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#pragma once

#include "dispatch_registration/dispatch-registry.h"

namespace ggml::hrx {

void register_amax_scale_dispatch(DispatchRegistryBuilder & registry);

} // namespace ggml::hrx
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#include "dispatch-glu.h"
#include "dispatch-mul-mat-id.h"
#include "dispatch-mul-mat.h"
#include "dispatch-amax-scale.h"
#include "dispatch-rmsnorm.h"
#include "dispatch-rope-set-rows.h"
#include "dispatch-scale.h"
Expand All @@ -28,6 +29,7 @@ void register_common_dispatches(DispatchRegistryBuilder & registry) {
register_glu_dispatches(registry);
register_mul_mat_id_dispatches(registry);
register_mul_mat_dispatches(registry);
register_amax_scale_dispatch(registry);
register_rope_set_rows_dispatches(registry);
register_scale_dispatch(registry);
register_unary_dispatch(registry);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,6 @@ static bool has_flash_attention_params(const GraphNode & node) {
(params->prec == GGML_PREC_DEFAULT || params->prec == GGML_PREC_F32);
}

static size_t attention_mask_byte_count(int64_t query_token_count, int64_t key_value_token_count) {
if (query_token_count <= 0 || key_value_token_count <= 0) {
return 0;
}
return static_cast<size_t>(query_token_count) * static_cast<size_t>(key_value_token_count) * sizeof(ggml_fp16_t);
}

static std::string to_config_value(int64_t value) {
return std::to_string(value);
}
Expand Down Expand Up @@ -144,8 +137,6 @@ struct FlashAttentionMatch {
const Value * mask = nullptr;
const Value * output = nullptr;
const GraphNode * output_layout = nullptr;
ValueId mask_binding_value;
size_t mask_binding_bytes = 0;
int64_t query_token_count = 0;
int64_t key_value_token_count = 0;
int64_t query_head_count = 0;
Expand Down Expand Up @@ -181,7 +172,6 @@ struct DecodeSplitFlashAttentionMatch {
};

static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & graph,
const CommandPlan & plan,
const GraphNode * node) {
FlashAttentionMatch match;
if (node == nullptr || node->op != GGML_OP_FLASH_ATTN_EXT || node->inputs.size() != 4) {
Expand Down Expand Up @@ -229,20 +219,9 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra
return {};
}

int64_t key_value_token_count = key_value_capacity;
ValueId mask_binding_value = mask->id;
size_t mask_binding_bytes = mask->byte_count;
bool mask_binding_is_alternate = false;
const size_t compact_mask_bytes = attention_mask_byte_count(query_token_count, query_token_count);
const auto * compact_mask = find_alternate_value(plan, mask->id, GGML_TYPE_F16, compact_mask_bytes);
const bool mask_is_compact = mask->ne[0] == query_token_count;
const bool mask_is_capacity = mask->ne[0] == key_value_capacity;
if (compact_mask != nullptr && mask_is_capacity && mask->ne[0] > query_token_count) {
key_value_token_count = query_token_count;
mask_binding_value = compact_mask->alternate_value;
mask_binding_bytes = compact_mask->byte_count;
mask_binding_is_alternate = true;
} else if (!mask_is_compact && !mask_is_capacity) {
const int64_t key_value_token_count = key_value_capacity;
const bool mask_is_capacity = mask->ne[0] == key_value_capacity;
if (!mask_is_capacity) {
return {};
}
if (!is_supported_key_value_token_count(key_value_token_count)) {
Expand All @@ -255,7 +234,7 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra
!has_output_layout(*output, query_head_count, value_head_size)) {
return {};
}
if (!mask_binding_is_alternate && !has_mask_layout(*mask, key_value_token_count)) {
if (!has_mask_layout(*mask, key_value_token_count)) {
return {};
}

Expand All @@ -265,8 +244,6 @@ static FlashAttentionMatch match_flash_attention_f32_f16(const Graph & gra
match.mask = mask;
match.output = output;
match.output_layout = find_single_layout_alias_consumer(graph, output->id);
match.mask_binding_value = mask_binding_value;
match.mask_binding_bytes = mask_binding_bytes;
match.query_token_count = query_token_count;
match.key_value_token_count = key_value_token_count;
match.query_head_count = query_head_count;
Expand Down Expand Up @@ -425,7 +402,7 @@ static DispatchBinding prepare_flash_attention_value(const DispatchMatchContext
}

static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) {
const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node);
const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node);
if (!match.matched() || match.output_layout == nullptr || match.output_layout->op != GGML_OP_RESHAPE) {
return false;
}
Expand Down Expand Up @@ -503,7 +480,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con
dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count });
dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count });
dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel));
dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes });
dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count });
dispatch.bindings.push_back({ raw_gate->id, 0, raw_gate->byte_count });
dispatch.bindings.push_back({ output->id, 0, output->byte_count });
dispatch_match.dispatches.push_back(std::move(dispatch));
Expand All @@ -512,7 +489,7 @@ static bool match_flash_attention_gate_dispatch(const DispatchMatchContext & con

static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext & context,
DispatchMatch & dispatch_match) {
const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.plan, context.root_node);
const FlashAttentionMatch match = match_flash_attention_f32_f16(context.graph, context.root_node);
if (!match.matched()) {
return false;
}
Expand Down Expand Up @@ -540,7 +517,7 @@ static bool match_flash_attention_f32_f16_dispatch(const DispatchMatchContext &
dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count });
dispatch.bindings.push_back({ match.key->id, 0, match.key->byte_count });
dispatch.bindings.push_back(prepare_flash_attention_value(context, match, dispatch_match, dispatch.kernel));
dispatch.bindings.push_back({ match.mask_binding_value, 0, match.mask_binding_bytes });
dispatch.bindings.push_back({ match.mask->id, 0, match.mask->byte_count });
dispatch.bindings.push_back({ match.query->id, 0, match.query->byte_count });
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });

Expand Down
Loading
Loading