Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
92c1f6c
hrx: prefer row64 weights for dense Qwen3
AaronStGeorge Nov 11, 2026
9c27b0a
hrx: support row64 tied embedding and vocabulary projection
AaronStGeorge Nov 11, 2026
fdabd06
test: cover Q6 row64 short residual numerics (lemonade-ws-aj0j.6)
AaronStGeorge Nov 11, 2026
1a7db48
ggml-hrx: use M64 Q6 row64 tiles for short Qwen3 prefills
AaronStGeorge Nov 11, 2026
4f114e8
ggml-hrx: reuse Q6 row64 decode loads
AaronStGeorge Nov 11, 2026
834ebe5
hrx: use generic gfx11 row64 Q6 projection
AaronStGeorge Nov 16, 2026
a27a4be
hrx: adapt row64 additions to upstream vector splat syntax
AaronStGeorge Nov 16, 2026
3649b96
hrx: package row64 performance reproduction for gfx11
AaronStGeorge Nov 16, 2026
6837ad7
hrx: resolve current Loom tools in performance runner
AaronStGeorge Nov 16, 2026
bf36852
hrx: record rebased gfx1100 performance and projection gap
AaronStGeorge Nov 16, 2026
13df855
hrx: remove row64 investigation scaffolding and cover T17 endpoints
AaronStGeorge Nov 16, 2026
caf5626
hrx: reuse shared row64 matvec for single-token Qwen endpoints
AaronStGeorge Nov 16, 2026
f74241b
hrx: consolidate native and row64 Q6 Q8 decode motif (lemonade-ws-s9tq)
AaronStGeorge Nov 16, 2026
e37b234
hrx: consolidate Q6 Q8 endpoint ABI and vector entry body
AaronStGeorge Nov 16, 2026
02a8bcb
hrx: separate native and row64 Q6 Q8 motifs
AaronStGeorge Nov 16, 2026
53dcc97
test: restore native Q6/Q8 entry coverage
AaronStGeorge Nov 16, 2026
0bad06b
hrx: rename general vector matmul entry
AaronStGeorge Nov 16, 2026
6635381
hrx: preserve existing vector kernel names
AaronStGeorge Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -103,12 +103,21 @@ struct GetRowsMatch {
int64_t token_count = 0;
int64_t row_count = 0;
int64_t hidden_size = 0;
std::string weight_layout = kNativeWeightLayout;

bool matched() const {
return ids != nullptr && weight != nullptr && output != nullptr && weight_format_value >= 0;
}
};

static DispatchBinding get_rows_weight_binding(const GetRowsMatch & match) {
if (match.weight_layout == kNativeWeightLayout) {
return { match.weight->id, 0, match.weight->byte_count };
}
return { match.weight->id, 0, match.weight->byte_count, match.weight_layout, match.weight->type,
match.hidden_size, match.row_count, match.weight->byte_count };
}

struct GetRowsRmsNormMatch {
GetRowsMatch rows;
const GraphNode * rms_node = nullptr;
Expand Down Expand Up @@ -153,7 +162,7 @@ static void add_common_integer_parameters(Dispatch & dispatch, const GetRowsMatc

static void add_primary_bindings(Dispatch & dispatch, const GetRowsMatch & match) {
dispatch.bindings.push_back({ match.ids->id, 0, match.ids->byte_count });
dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count });
dispatch.bindings.push_back(get_rows_weight_binding(match));
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });
}

Expand Down Expand Up @@ -256,8 +265,10 @@ static bool match_get_rows_scalar_3d_f32_dispatch(const DispatchMatchContext & c
return true;
}

static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * node) {
static GetRowsMatch match_get_rows_f32(const DispatchMatchContext & context) {
GetRowsMatch match;
const Graph & graph = context.graph;
const GraphNode * node = context.root_node;
if (node == nullptr || node->op != GGML_OP_GET_ROWS || node->inputs.size() != 2) {
return match;
}
Expand Down Expand Up @@ -293,12 +304,23 @@ static GetRowsMatch match_get_rows_f32(const Graph & graph, const GraphNode * no
match.token_count = token_count;
match.row_count = row_count;
match.hidden_size = hidden_size;
if (context.row64_weight_policy && weight->alias_source.value < 0 && row_count % 64 == 0 &&
hidden_size % 256 == 0 &&
dispatch_weight_layout_eligible(*weight)) {
if (weight->type == GGML_TYPE_Q4_K) {
match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q4KRow64);
match.weight_layout = kQ4KPackedK256Row64Layout;
} else if (weight->type == GGML_TYPE_Q6_K) {
match.weight_format_value = common_mul_mat_format_config_value(CommonMulMatWeightFormat::Q6KRow64);
match.weight_layout = kQ6KPackedK256Row64ScaleRowLayout;
}
}
return match;
}

static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & context) {
GetRowsScaleMatch match;
match.rows = match_get_rows_f32(context.graph, context.root_node);
match.rows = match_get_rows_f32(context);
if (!match.rows.matched() || !context.graph.has_index()) {
return {};
}
Expand Down Expand Up @@ -345,7 +367,7 @@ static GetRowsScaleMatch match_get_rows_scale_f32(const DispatchMatchContext & c

static GetRowsRmsNormMatch match_get_rows_rmsnorm_binary(const DispatchMatchContext & context) {
GetRowsRmsNormMatch match;
match.rows = match_get_rows_f32(context.graph, context.root_node);
match.rows = match_get_rows_f32(context);
if (!match.rows.matched() || !context.graph.has_index() || match.rows.hidden_size < 128 ||
match.rows.hidden_size > 32768 || match.rows.hidden_size % 128 != 0) {
return {};
Expand Down Expand Up @@ -433,7 +455,7 @@ static Dispatch make_get_rows_next_dispatch(const GetRowsMatch & match,
}

static bool match_get_rows_f32_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) {
const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node);
const GetRowsMatch match = match_get_rows_f32(context);
if (!match.matched()) {
return false;
}
Expand All @@ -458,7 +480,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte
dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.scale", to_config_value(match.scale));
dispatch.kernel.compile_parameters.emplace("ggml.get_rows_scale_f32.bias", to_config_value(match.bias));
dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count });
dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count });
dispatch.bindings.push_back(get_rows_weight_binding(match.rows));
dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count });
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });

Expand All @@ -469,7 +491,7 @@ static bool match_get_rows_scale_f32_dispatch(const DispatchMatchContext & conte
}

static bool match_get_rows_f32_next_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) {
const GetRowsMatch match = match_get_rows_f32(context.graph, context.root_node);
const GetRowsMatch match = match_get_rows_f32(context);
if (!match.matched()) {
return false;
}
Expand Down Expand Up @@ -558,7 +580,7 @@ static bool match_get_rows_rmsnorm_binary_dispatch(const DispatchMatchContext &
add_common_compile_parameters(dispatch, match.rows);
dispatch.kernel.compile_parameters.emplace("ggml.get_rows_rmsnorm.rms_epsilon", to_config_value(match.epsilon));
dispatch.bindings.push_back({ match.rows.ids->id, 0, match.rows.ids->byte_count });
dispatch.bindings.push_back({ match.rows.weight->id, 0, match.rows.weight->byte_count });
dispatch.bindings.push_back(get_rows_weight_binding(match.rows));
dispatch.bindings.push_back({ match.norm_weight->id, 0, match.norm_weight->byte_count });
dispatch.bindings.push_back({ match.rows.output->id, 0, match.rows.output->byte_count });
dispatch.bindings.push_back({ match.normalized_output->id, 0, match.normalized_output->byte_count });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,8 @@ static constexpr int64_t kMulMatVectorPackedMaxOutputSize = 26
static constexpr int64_t kMulMatVectorSelectedOutputsMaxOutputSize = 262144;
static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillWave32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_wave32");
static constexpr KernelCatalogRef kMulMatQ6KF32WmmaPrefillM64Wave32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f32_wmma_prefill_m64_wave32");
static constexpr KernelCatalogRef kMulMatQ6KF16WmmaPrefillWave32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_q6_k_f16_wmma_prefill_wave32");
static constexpr KernelCatalogRef kSelectSymmetricI4K32GroupsKernel =
Expand Down Expand Up @@ -903,10 +905,13 @@ static bool match_q6_k_prefill_wave32_dispatch(const DispatchMatchContext & cont
return false;
}

// Short prompts use masked M128 tiles without a separate activation copy.
// Short prompts use masked WMMA tiles without a separate activation copy.
if (short_row64) {
// M64 halves reduce register pressure at these measured short shapes.
const bool use_m64 = match.output_size == 1024 && (match.input_size == 1024 || match.input_size == 3072);
return build_q6_k_prefill_wave32_dispatch(match, match.input->id, match.input->byte_count,
kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight,
use_m64 ? kMulMatQ6KF32WmmaPrefillM64Wave32Kernel :
kMulMatQ6KF32WmmaPrefillWave32Kernel, row64_weight,
context.root_index, dispatch_match);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ static constexpr KernelCatalogRef kQwenDenseLinearQ4KQ8NextQ8Kernel =
GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_dense_linear_q4k_q8_1_x4_next_q8");
static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel =
GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4");
static constexpr KernelCatalogRef kMulMatVectorF32F32Kernel =
GGML_HRX_KERNEL_REF("loom_libs", "ggml_mul_mat_vector_f32_f32");

static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size;
static constexpr int64_t kQwenVocabularyCount = 151936;
Expand Down Expand Up @@ -336,32 +338,49 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context,
output->ne[2] == 1 && output->ne[3] == 1;
}

static void build_qwen_matmul_dispatch(const QwenMatmulMatch & match,
DispatchMatch & dispatch_match,
size_t root_index) {
Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output,
const DispatchBinding & input, bool row64_weight_policy) {
const int64_t input_size = weight.ne[0];
const int64_t output_size = weight.ne[1];
const bool row64_weight = row64_weight_policy && weight.alias_source.value < 0 &&
dispatch_weight_layout_eligible(weight);
Dispatch dispatch;
dispatch.kernel = make_kernel_specialization(match.kernel);
dispatch.kernel.integer_parameters.emplace("token_count", match.token_count);
dispatch.kernel.integer_parameters.emplace("input_size", match.input_size);
dispatch.kernel.integer_parameters.emplace("output_size", match.output_size);
dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity",
to_config_value(match.token_count));
dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity",
to_config_value(match.output_size));
dispatch.bindings.push_back({ match.input_value, 0, match.input_bytes });
dispatch.bindings.push_back({ match.weight->id, 0, match.weight->byte_count });
dispatch.bindings.push_back({ match.output->id, 0, match.output->byte_count });

dispatch_match.covered_nodes.push_back(root_index);
dispatch_match.dispatches.push_back(std::move(dispatch));
dispatch.kernel =
make_kernel_specialization(row64_weight ? kMulMatVectorF32F32Kernel : kGgmlLinearQ6KQ8_1X4Kernel);
dispatch.kernel.integer_parameters.emplace("token_count", 1);
dispatch.kernel.compile_parameters.emplace("ggml.workload.token_capacity", "1");
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.input_size", to_config_value(input_size));
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_size", to_config_value(output_size));
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_accumulation", "0");
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.output_unary_op",
std::to_string(unary_kind_config_value(UnaryKind::Identity)));
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.publish_format", "0");
dispatch.kernel.compile_parameters.emplace(
"ggml.matmul.vector.weight_format",
to_config_value(common_mul_mat_format_config_value(
row64_weight ? CommonMulMatWeightFormat::Q6KRow64 : CommonMulMatWeightFormat::Q6K)));
dispatch.kernel.compile_parameters.emplace("ggml.matmul.vector.activation_format",
std::to_string(GGML_TYPE_Q8_1));
dispatch.bindings.push_back(input);
if (row64_weight) {
dispatch.bindings.push_back({ weight.id, 0, weight.byte_count, kQ6KPackedK256Row64ScaleRowLayout,
weight.type, input_size, output_size, weight.byte_count });
} else {
dispatch.bindings.push_back({ weight.id, 0, weight.byte_count });
}
dispatch.bindings.push_back({ output.id, 0, output.byte_count });
dispatch.bindings.push_back({ output.id, 0, output.byte_count });
return dispatch;
}

static bool match_qwen_q6k_q8_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) {
const QwenMatmulMatch match = match_qwen_q6k_q8_matmul(context.graph, context.root_node, context.plan);
if (!match.matched()) {
return false;
}
build_qwen_matmul_dispatch(match, dispatch_match, context.root_index);
dispatch_match.covered_nodes.push_back(context.root_index);
dispatch_match.dispatches.push_back(make_qwen_q6k_q8_projection_dispatch(
*match.weight, *match.output, { match.input_value, 0, match.input_bytes }, context.row64_weight_policy));
return true;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ bool qwen_accepts_q6k_q8_input(const DispatchMatchContext & context,
const GraphNode & consumer,
const Value & input);

// The endpoint Q8 projection is a single-token path; prefill uses common matmul dispatch.
Dispatch make_qwen_q6k_q8_projection_dispatch(const Value & weight, const Value & output,
const DispatchBinding & input, bool row64_weight_policy);

void register_qwen_matmul_dispatches(DispatchRegistryBuilder & registry);

} // namespace ggml::hrx
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
#include "dispatch-qwen-rmsnorm.h"
#include "dispatch-qwen-matmul.h"

#include "dispatch-llm-profiles.h"
#include "../common/dispatch-activation-publication.h"
Expand All @@ -17,8 +18,6 @@ namespace {

static constexpr KernelCatalogRef kQwenRmsNormF32QuantizeQ8_1X4Kernel =
GGML_HRX_KERNEL_REF("qwen3_moe", "qwen3_moe_rmsnorm_f32_quantize_q8_1_x4");
static constexpr KernelCatalogRef kGgmlLinearQ6KQ8_1X4Kernel =
GGML_HRX_KERNEL_REF("qwen3_moe", "ggml_linear_q6k_q8_1_x4");
static constexpr float kQwenRmsNormEpsilon = kQwen30BMoeDispatchProfile.rms_norm_epsilon;
static constexpr int64_t kQwenHiddenSize = kQwen30BMoeDispatchProfile.hidden_size;
static constexpr int64_t kQwenVocabularyCount = 151936;
Expand Down Expand Up @@ -187,7 +186,7 @@ static QwenEndpointProjectionMatch match_qwen_endpoint_projection(const Graph &
}
if (input->id != match.output->id || weight->type != GGML_TYPE_Q6_K || output->type != GGML_TYPE_F32 ||
!weight->contiguous || !input->contiguous || !output->contiguous || match.hidden_size != kQwenHiddenSize ||
!is_supported_token_count(match.token_count) || weight->ne[0] != match.hidden_size ||
match.token_count != 1 || weight->ne[0] != match.hidden_size ||
weight->ne[1] != kQwenVocabularyCount || output->ne[0] != kQwenVocabularyCount ||
output->ne[1] != match.token_count || output->ne[2] != 1 || output->ne[3] != 1) {
return {};
Expand Down Expand Up @@ -284,18 +283,9 @@ static bool match_qwen_rmsnorm_f32_quantize_q8_1_x4_dispatch(const DispatchMatch
rms_dispatch.bindings.push_back({ rms_match.output->id, 0, rms_match.output->byte_count });
rms_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count });

Dispatch projection_dispatch;
projection_dispatch.kernel = make_kernel_specialization(kGgmlLinearQ6KQ8_1X4Kernel);
projection_dispatch.kernel.integer_parameters.emplace("token_count", rms_match.token_count);
projection_dispatch.kernel.integer_parameters.emplace("input_size", rms_match.hidden_size);
projection_dispatch.kernel.integer_parameters.emplace("output_size", kQwenVocabularyCount);
projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity",
to_config_value(rms_match.token_count));
projection_dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity",
to_config_value(kQwenVocabularyCount));
projection_dispatch.bindings.push_back({ q8_value, 0, q8_byte_count });
projection_dispatch.bindings.push_back({ projection_match.weight->id, 0, projection_match.weight->byte_count });
projection_dispatch.bindings.push_back({ projection_match.output->id, 0, projection_match.output->byte_count });
Dispatch projection_dispatch = make_qwen_q6k_q8_projection_dispatch(
*projection_match.weight, *projection_match.output, { q8_value, 0, q8_byte_count },
context.row64_weight_policy);

match.covered_nodes.push_back(rms_match.rms_node_index);
match.covered_nodes.push_back(rms_match.mul_node_index);
Expand Down
Loading
Loading