From 2cbca389f515df455412dafa07e2b1359cc72a6d Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 09:02:32 +0800 Subject: [PATCH 001/286] dflash: copy hidden features via device copies (#31506) --- common/common.cpp | 16 ++++ common/common.h | 5 ++ common/speculative.cpp | 39 +++++++++ src/llama-batch.cpp | 64 +++++++++++++- src/llama-batch.h | 5 ++ src/llama-context.cpp | 165 +++++++++++++++++++++++++++++++++++- src/llama-context.h | 25 ++++++ src/llama-cparams.h | 1 + src/llama-ext.h | 13 +++ src/llama-graph.cpp | 57 ++++++++++++- src/llama-graph.h | 28 +++++- src/llama-kv-cache-dsv4.cpp | 1 + src/llama-model.cpp | 2 + src/models/dflash.cpp | 45 +++++----- 14 files changed, 441 insertions(+), 25 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 04e348805ba3..95a2dceef65d 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -2275,6 +2275,16 @@ int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq return size() - 1; } +int32_t common_batch::add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output) { + token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} }; + t.embd_row = row; + for (int32_t j = 0; j < n_pos; ++j) { + t.pos[j] = pos[j]; + } + tokens.push_back(t); + return size() - 1; +} + llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) { GGML_ASSERT(batch && "common_batch was not initialized with a context"); GGML_ASSERT(off >= 0 && n >= 0 && off + n <= size()); @@ -2295,6 +2305,12 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) { if (t.embd.data && !llama_batch_ext_set_embd_token(res, idx, t.embd)) { GGML_ABORT("%s: failed to set the embedding of token %d at index %d\n", __func__, t.id, i); } + } else if (t.embd_row >= 0) { + idx = llama_batch_ext_add_embd_row(res, t.seq_id, t.embd_row); + if (idx < 0) { + GGML_ABORT("%s: failed to add embd row %d at index %d (error %d, n = %d)\n", __func__, t.embd_row, i, idx, n); + } + llama_batch_ext_set_pos(res, idx, t.pos.data()); } else { idx = llama_batch_ext_add_embd(res, t.seq_id, t.embd); if (idx < 0) { diff --git a/common/common.h b/common/common.h index d0e33fb7500d..e970fb39b25d 100644 --- a/common/common.h +++ b/common/common.h @@ -1096,6 +1096,7 @@ struct common_batch { llama_embd state; // non-owning view of the data passed to set_embd_state(), data == NULL if none std::vector seq_ids_extra; // see add_seq() int32_t decision_order = 0; // see llama_batch_ext_set_decision_order() + int32_t embd_row = -1; // see add_embd_row(), -1 if none }; std::vector tokens; // mirror of the entries, tokens[i] describes batch index i @@ -1138,6 +1139,10 @@ struct common_batch { // pos points to n_pos positions int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output); + // add an entry whose embedding is row `row` of the layer input device buffer of the context's ctx_other + // pos points to n_pos positions + int32_t add_embd_row(int32_t row, const llama_pos * pos, llama_seq_id seq_id, bool output); + int32_t size() const { return (int32_t) tokens.size(); } }; diff --git a/common/speculative.cpp b/common/speculative.cpp index 4305ac6acb41..14fee2b2c501 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1027,6 +1027,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices uint32_t target_layer_ids_n = 0; + // inject target features from the target's device buffer instead of host copies (env LLAMA_DFLASH_DEV=0 disables) + bool use_dev_features = false; + common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq, common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH) : common_speculative_impl(type, n_seq, params.draft.n_max) @@ -1141,6 +1144,23 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); } + // the device buffer keeps layers in ascending id order, the fc input expects target_layer_ids order + { + const char * env = getenv("LLAMA_DFLASH_DEV"); + const bool ascending = std::adjacent_find(target_layer_ids, target_layer_ids + target_layer_ids_n, + [](int32_t a, int32_t b) { return a >= b; }) == target_layer_ids + target_layer_ids_n; + + use_dev_features = !is_lilicorr && ascending && (env == nullptr || atoi(env) != 0); + if (use_dev_features) { + llama_set_embeddings_layer_inp_dev(ctx_tgt, true); + use_dev_features = llama_supports_embd_row(ctx_dft); + if (!use_dev_features) { + llama_set_embeddings_layer_inp_dev(ctx_tgt, false); + } + } + LOG_INF("%s: DFlash target features via %s\n", __func__, use_dev_features ? "device buffer" : "host copies"); + } + // DFlash2 reads its selector lattice from h_nextn and never consumes raw logits. llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ !is_dflash2 && !is_lilicorr); llama_set_causal_attn(ctx_dft, causal_attn); // DFlash needs non-causal attention unless the model says otherwise @@ -1233,6 +1253,25 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); + if (use_dev_features) { + // the features are the target batch rows in the target's device buffer + batch_inject.clear(); + for (int32_t i = 0; i < n_chunk; ++i) { + const int32_t row = i_batch_beg[seq_id] + offset + i; + const llama_pos p = batch_in.tokens[row].pos[0]; + const llama_pos pos_arr[4] = { p, p, p, 0 }; + batch_inject.add_embd_row(row, pos_arr, seq_id, false); + } + + const int32_t rc = llama_process(ctx_dft, LLAMA_PROCESS_TYPE_DECODE, batch_inject.get()); + if (rc != 0) { + LOG_ERR("%s: llama_process(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) offset); + return false; + } + continue; + } + // gather target features per extract layer; the fused decode encodes and // injects them into the K/V cache at the target positions features_buf.resize((size_t) n_chunk * n_embd_enc); diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp index 321aa9d23647..f84c6d50be61 100644 --- a/src/llama-batch.cpp +++ b/src/llama-batch.cpp @@ -52,6 +52,7 @@ bool llama_batch_allocr::init( // an entry can carry a token id, a token embedding, or both // all entries must carry the same combination, or be a mix of token and embd entries // a state embedding (e.g. MTP hook batches) is set on all entries or on none + // an embd row (embd read from a device buffer) is set on all entries or on none, with no token id or embedding // int32_t n_tok_only = 0; @@ -59,12 +60,24 @@ bool llama_batch_allocr::init( int32_t n_both = 0; const bool has_state = batch_inp.tokens[0].has_state; + const bool has_row = batch_inp.tokens[0].embd_row >= 0; for (int32_t i = 0; i < n_tok; ++i) { const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL; const bool is_emb = batch_inp.tokens[i].has_embd; + const bool is_row = batch_inp.tokens[i].embd_row >= 0; - if (!is_tok && !is_emb) { + if (is_row != has_row) { + LLAMA_LOG_ERROR("%s: all entries in the batch must have the same embd row presence\n", __func__); + return false; + } + + if (is_row) { + if (is_tok || is_emb) { + LLAMA_LOG_ERROR("%s: entry %d has an embd row and a token id or embedding\n", __func__, i); + return false; + } + } else if (!is_tok && !is_emb) { LLAMA_LOG_ERROR("%s: entry %d has neither a token id nor an embedding\n", __func__, i); return false; } @@ -137,6 +150,13 @@ bool llama_batch_allocr::init( state_vec = batch_inp.state; } + if (has_row) { + embd_row_vec.resize(n_tok); + for (int32_t i = 0; i < n_tok; ++i) { + embd_row_vec[i] = batch_inp.tokens[i].embd_row; + } + } + // // build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i // token entry: [p, p, p, 0] (M-RoPE text position) @@ -306,6 +326,7 @@ bool llama_batch_allocr::init( /*.token =*/ batch.token, /*.embd =*/ batch.embd, /*.embd_state =*/ state_vec.empty() ? nullptr : state_vec.data(), + /*.embd_row =*/ embd_row_vec.empty() ? nullptr : embd_row_vec.data(), /*.pos =*/ batch.pos, /*.n_seq_id =*/ batch.n_seq_id, /*.seq_id =*/ batch.seq_id, @@ -531,6 +552,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t /*.token =*/ udata->token.data(), /*.embd =*/ nullptr, /*.embd_state =*/ nullptr, + /*.embd_row =*/ nullptr, /*.pos =*/ udata->pos.data(), /*.n_seq_id =*/ udata->n_seq_id.data(), /*.seq_id =*/ udata->seq_id.data(), @@ -838,6 +860,7 @@ void llama_batch_allocr::clear() { embd_vec .clear(); is_embd_vec .clear(); state_vec .clear(); + embd_row_vec.clear(); seq_id_data .clear(); pos .clear(); n_seq_id .clear(); @@ -881,6 +904,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens); const bool use_embd = batch.embd && !(mixed_batch && n_embd_rows == 0); const bool has_state = !state_vec.empty(); + const bool has_row = !embd_row_vec.empty(); const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0; const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0; @@ -889,6 +913,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u udata->token .resize(n_tokens); udata->embd .resize(n_embd_all); udata->embd_state.resize(n_state_all); + udata->embd_row .resize(has_row ? n_tokens : 0); udata->pos .resize(n_pos_all); udata->n_seq_id .resize(n_tokens); udata->seq_id .resize(n_tokens); @@ -920,6 +945,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float)); } + if (has_row) { + udata->embd_row[i] = embd_row_vec[idxs[i]]; + } + for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) { udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]]; } @@ -967,6 +996,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u /*.token =*/ use_token ? udata->token.data() : nullptr, /*.embd =*/ use_embd ? udata->embd.data() : nullptr, /*.embd_state =*/ has_state ? udata->embd_state.data() : nullptr, + /*.embd_row =*/ has_row ? udata->embd_row.data() : nullptr, /*.pos =*/ udata->pos.data(), /*.n_seq_id =*/ udata->n_seq_id.data(), /*.seq_id =*/ udata->seq_id.data(), @@ -1019,6 +1049,7 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) { LLAMA_LOG_DEBUG("%s: token = %p\n", __func__, (void *) ubatch.token); LLAMA_LOG_DEBUG("%s: embd = %p\n", __func__, (void *) ubatch.embd); LLAMA_LOG_DEBUG("%s: embd_state = %p\n", __func__, (void *) ubatch.embd_state); + LLAMA_LOG_DEBUG("%s: embd_row = %p\n", __func__, (void *) ubatch.embd_row); LLAMA_LOG_DEBUG("%s: pos = %p\n", __func__, (void *) ubatch.pos); LLAMA_LOG_DEBUG("%s: n_seq_id = %p\n", __func__, (void *) ubatch.n_seq_id); LLAMA_LOG_DEBUG("%s: seq_id = %p\n", __func__, (void *) ubatch.seq_id); @@ -1303,6 +1334,26 @@ bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) { return true; } +bool llama_batch_ext::set_token_embd_row(int32_t idx, int32_t row) { + if (idx < 0 || idx >= (int32_t) tokens.size()) { + return false; + } + if (row < 0) { + return false; + } + + token & t = tokens[idx]; + + if (t.id != LLAMA_TOKEN_NULL || t.has_embd || t.embd_row >= 0) { + LLAMA_LOG_ERROR("%s: entry %d already has a token id, an embedding or an embd row\n", __func__, idx); + return false; + } + + t.embd_row = row; + + return true; +} + bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) { if (idx < 0 || idx >= (int32_t) tokens.size()) { return false; @@ -1377,6 +1428,17 @@ int32_t llama_batch_ext_add_embd(llama_batch_ext * batch, llama_seq_id seq_id, l return idx; } +int32_t llama_batch_ext_add_embd_row(llama_batch_ext * batch, llama_seq_id seq_id, int32_t row) { + int32_t idx = batch->add_token(seq_id); + if (idx < 0) { + return idx; + } + if (!batch->set_token_embd_row(idx, row)) { + return -2; + } + return idx; +} + bool llama_batch_ext_add_seq(llama_batch_ext * batch, int32_t idx, llama_seq_id seq_id) { return batch->add_seq(idx, seq_id); } diff --git a/src/llama-batch.h b/src/llama-batch.h index e1e43979b540..d2361396cd49 100644 --- a/src/llama-batch.h +++ b/src/llama-batch.h @@ -52,6 +52,7 @@ struct llama_ubatch { llama_token * token; // [n_tokens] | i | id, token float * embd; // [n_embd, n_tokens] | i | embd float * embd_state; // [n_embd_state, n_tokens] | i | hidden state carried over from a previous stage (e.g. MTP) + int32_t * embd_row; // [n_tokens] | i | embd is this row of ctx_other's layer input device buffer llama_pos * pos; // [n_tokens*n_pos] | i | pos int32_t * n_seq_id; // [n_tokens] | i | - llama_seq_id ** seq_id; // [n_tokens] | s | s0, s1, seq_id @@ -65,6 +66,7 @@ struct llama_ubatch { std::vector token; std::vector embd; std::vector embd_state; + std::vector embd_row; std::vector pos; std::vector n_seq_id; std::vector seq_id; // these point into the seq_id_data below @@ -114,6 +116,7 @@ struct llama_batch_ext { size_t embd_off = 0; // index offset in the embd array bool has_state = false; // whether state_off is set size_t state_off = 0; // index offset in the state array + int32_t embd_row = -1; // row of ctx_other's layer input device buffer used as embd, -1 if none bool output = false; // TODO: have dedicated output flags int32_t decision_order = 0; // see llama_batch_ext_set_decision_order() std::unordered_set seq_ids; @@ -145,6 +148,7 @@ struct llama_batch_ext { bool set_token_id(int32_t idx, llama_token id); bool set_token_embd(int32_t idx, llama_embd embd_in); bool set_token_state(int32_t idx, llama_embd state_in); + bool set_token_embd_row(int32_t idx, int32_t row); bool set_token_pos(int32_t idx, const llama_pos * pos_in); bool set_output(int32_t idx, bool output_last); bool set_decision_order(int32_t idx, int32_t order); @@ -223,6 +227,7 @@ class llama_batch_allocr { std::vector embd_vec; // owned embeddings built from llama_batch_ext std::vector is_embd_vec; // mixed batch only (= 1 if embd, 0 if text token) std::vector state_vec; // owned state embeddings built from llama_batch_ext, llama_batch has no slot for them + std::vector embd_row_vec; // owned embd rows built from llama_batch_ext, llama_batch has no slot for them std::vector seq_id_data; // flat storage for seq_id pointers below std::vector pos; diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 66b724c18b62..0cd795a5690b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -222,6 +222,7 @@ llama_context::llama_context( // +1: id n_layer() taps the output of the last layer ("input" of the head) cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false); + cparams.embeddings_layer_inp_dev = false; embd_layer_inp.resize(hparams.n_layer() + 1); cparams.ctx_type = params.ctx_type; @@ -257,6 +258,9 @@ llama_context::llama_context( throw std::runtime_error(model.arch_name() + " requires ctx_other to be set (this warning is normal during memory fitting)"); } cparams.ctx_other = params.ctx_other; + } else if (model.arch == LLM_ARCH_DFLASH) { + // used to read the target's layer input device buffer + cparams.ctx_other = params.ctx_other; } } @@ -1396,6 +1400,109 @@ void llama_context::set_embeddings_layer_inp(uint32_t lid, bool enable) { // note: without this reserve, the draft acceptance drops to zero. not sure why - this is unexpected sched_need_reserve = true; + + if (cparams.embeddings_layer_inp_dev) { + layer_inp_dev_alloc(); + } +} + +void llama_context::set_embeddings_layer_inp_dev(bool value) { + LLAMA_LOG_DEBUG("%s: value = %d\n", __func__, value); + + if (cparams.embeddings_layer_inp_dev != value) { + sched_need_reserve = true; + } + + cparams.embeddings_layer_inp_dev = value; + + layer_inp_dev_alloc(); +} + +bool llama_context::supports_embd_row() const { + // the DFlash/DSpark decoder graphs read the device buffer + if (model.arch != LLM_ARCH_DFLASH) { + return false; + } + + const ggml_tensor * src = cparams.ctx_other ? cparams.ctx_other->get_layer_inp_dev() : nullptr; + if (src == nullptr) { + return false; + } + + // one of our backends must be able to read the buffer, e.g. not a draft on another GPU than the target output + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(src->buffer); + for (const auto & backend : backends) { + if (ggml_backend_supports_buft(backend.get(), buft)) { + return true; + } + } + + return false; +} + +ggml_backend_t llama_context::backend_from_dev(ggml_backend_dev_t dev) const { + for (const auto & backend : backends) { + if (ggml_backend_get_device(backend.get()) == dev) { + return backend.get(); + } + } + return nullptr; +} + +void llama_context::layer_inp_dev_alloc() { + synchronize(); + + layer_inp_dev = nullptr; + layer_inp_dev_dev = nullptr; + layer_inp_backend = nullptr; + buf_layer_inp_dev.reset(); + ctx_layer_inp_dev.reset(); + + if (!cparams.embeddings_layer_inp_dev) { + return; + } + + int64_t n_layer_inp = 0; + for (bool enabled : cparams.embeddings_layer_inp) { + n_layer_inp += enabled ? 1 : 0; + } + if (n_layer_inp == 0) { + return; + } + + ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + ctx_layer_inp_dev.reset(ggml_init(params)); + + layer_inp_dev = ggml_new_tensor_2d(ctx_layer_inp_dev.get(), GGML_TYPE_F32, n_layer_inp*model.hparams.n_embd, cparams.n_batch); + ggml_set_name(layer_inp_dev, "layer_inp_dev"); + + layer_inp_dev_dev = model.dev_output(); + auto * buft = layer_inp_dev_dev ? ggml_backend_dev_buffer_type(layer_inp_dev_dev) : ggml_backend_cpu_buffer_type(); + + buf_layer_inp_dev.reset(ggml_backend_alloc_ctx_tensors_from_buft(ctx_layer_inp_dev.get(), buft)); + if (!buf_layer_inp_dev) { + // e.g. a large n_batch that does not fit next to the model, keep the host copies + LLAMA_LOG_WARN("%s: failed to allocate the layer input device buffer, using host copies\n", __func__); + cparams.embeddings_layer_inp_dev = false; + layer_inp_dev = nullptr; + layer_inp_dev_dev = nullptr; + ctx_layer_inp_dev.reset(); + return; + } + + LLAMA_LOG_INFO("%s: %10s layer input buffer size = %8.2f MiB (%d layers)\n", __func__, + ggml_backend_buffer_name(buf_layer_inp_dev.get()), ggml_backend_buffer_get_size(buf_layer_inp_dev.get())/1024.0/1024.0, (int) n_layer_inp); + + layer_inp_backend = layer_inp_dev_dev ? backend_from_dev(layer_inp_dev_dev) : nullptr; + if (layer_inp_backend && !ev_layer_inp_written) { + ev_layer_inp_written.reset(ggml_backend_event_new(layer_inp_dev_dev)); + ev_layer_inp_read.reset(ggml_backend_event_new(layer_inp_dev_dev)); + ev_layer_inp_read_recorded = false; + } } void llama_context::set_nextn_layer_offset(int32_t offset) { @@ -1627,6 +1734,11 @@ int llama_context::encode(const llama_batch_ext & batch_inp) { return -1; } + if (!batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0) { + LLAMA_LOG_ERROR("%s: embd row batches are not supported by encode\n", __func__); + return -1; + } + // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim const int64_t n_vocab = model.vocab.n_tokens(); @@ -1873,6 +1985,12 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { return -1; } + const bool has_embd_row = !batch_inp.tokens.empty() && batch_inp.tokens[0].embd_row >= 0; + if (has_embd_row && !supports_embd_row()) { + LLAMA_LOG_ERROR("%s: this context cannot read embd rows (needs a DFlash/DSpark decoder and a layer input device buffer in ctx_other)\n", __func__); + return -1; + } + const auto & vocab = model.vocab; const auto & hparams = model.hparams; @@ -2016,6 +2134,24 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { int64_t n_outputs_prev = 0; int64_t n_tokens_prev = 0; + // ordering for the layer input device buffer + llama_context * inject_src = has_embd_row ? cparams.ctx_other : nullptr; + ggml_backend_t inject_backend = nullptr; + if (inject_src) { + inject_backend = inject_src->ev_layer_inp_written ? backend_from_dev(inject_src->layer_inp_dev_dev) : nullptr; + if (inject_backend) { + // wait on the host: a device-side wait would stall our queue for the whole target batch, + // which trips the Metal command buffer timeout on long prompts + ggml_backend_event_synchronize(inject_src->ev_layer_inp_written.get()); + } else { + inject_src->synchronize(); + } + } + if (layer_inp_dev && ev_layer_inp_read && ev_layer_inp_read_recorded) { + // do not overwrite rows the reader has not consumed yet + ggml_backend_event_wait(layer_inp_backend, ev_layer_inp_read.get()); + } + do { const auto & ubatch = mctx->get_ubatch(); @@ -2202,6 +2338,18 @@ int llama_context::decode(const llama_batch_ext & batch_inp) { n_tokens_prev += ubatch.n_tokens; } while (mctx->next()); + if (layer_inp_dev && ev_layer_inp_written) { + ggml_backend_event_record(ev_layer_inp_written.get(), layer_inp_backend); + } + if (inject_src) { + if (inject_backend) { + ggml_backend_event_record(inject_src->ev_layer_inp_read.get(), inject_backend); + inject_src->ev_layer_inp_read_recorded = true; + } else { + synchronize(); + } + } + // set to total number of outputs in the batch, for use in llama_get_logits_ith n_outputs = n_outputs_all; @@ -2298,7 +2446,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { } for (bool enabled : cparams.embeddings_layer_inp) { - if (enabled) { + if (enabled && !cparams.embeddings_layer_inp_dev) { embd_layer_inp_float_count += (size_t) n_embd * n_batch; } } @@ -2370,7 +2518,7 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { offset += embd_nextn.size * sizeof(float); for (uint32_t il = 0; il < embd_layer_inp.size(); ++il) { - if (cparams.embeddings_layer_inp[il]) { + if (cparams.embeddings_layer_inp[il] && !cparams.embeddings_layer_inp_dev) { embd_layer_inp[il] = buffer_view{(float *) (base + offset), (size_t) n_embd * n_batch}; offset += embd_layer_inp[il].size * sizeof(float); } else { @@ -2426,6 +2574,9 @@ uint32_t llama_context::output_reserve(int32_t n_outputs) { bool llama_context::extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens) { bool extracted = false; + if (cparams.embeddings_layer_inp_dev) { + return extracted; + } for (uint32_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) { if (!cparams.embeddings_layer_inp[il]) { continue; @@ -2742,6 +2893,8 @@ llm_graph_params llama_context::graph_params( /*.moe_cache =*/ moe_cache.get(), /*.prec_policy =*/ &model.prec_policy, /*.hdmd =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd, + /*.layer_inp_dev =*/ layer_inp_dev, + /*.layer_inp_src =*/ supports_embd_row() ? cparams.ctx_other->get_layer_inp_dev() : nullptr, /*.samplers =*/ sampling.samplers, /*.n_outputs =*/ n_outputs, /*.cb =*/ graph_get_cb(), @@ -4342,6 +4495,14 @@ void llama_set_embeddings_nextn(llama_context * ctx, bool value, bool masked) { ctx->set_embeddings_nextn(value, masked); } +void llama_set_embeddings_layer_inp_dev(llama_context * ctx, bool value) { + ctx->set_embeddings_layer_inp_dev(value); +} + +bool llama_supports_embd_row(const llama_context * ctx) { + return ctx->supports_embd_row(); +} + void llama_set_embeddings_layer_inp(llama_context * ctx, uint32_t lid, bool value) { ctx->set_embeddings_layer_inp(lid, value); } diff --git a/src/llama-context.h b/src/llama-context.h index 6938fb103cde..39c5619df67d 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -93,6 +93,8 @@ struct llama_context { float * get_embeddings_layer_inp(uint32_t lid); + ggml_tensor * get_layer_inp_dev() const { return layer_inp_dev; } + llama_token * get_sampled_tokens() const; llama_token get_sampled_token_ith(int32_t idx); @@ -120,6 +122,10 @@ struct llama_context { void set_embeddings (bool value); void set_embeddings_nextn(bool value, bool masked); void set_embeddings_layer_inp(uint32_t lid, bool enable); + void set_embeddings_layer_inp_dev(bool value); + + // whether decode accepts entries whose embd is a row of ctx_other's layer input device buffer + bool supports_embd_row() const; void set_nextn_layer_offset(int32_t offset); void set_causal_attn(bool value); void set_warmup(bool value); @@ -243,6 +249,11 @@ struct llama_context { // from backend into host-side embd_layer_inp buffers bool extract_layer_inputs(const llm_graph_result * res, size_t token_offset, size_t n_tokens); + // (re)allocate the device buffer for cparams.embeddings_layer_inp_dev + void layer_inp_dev_alloc(); + + ggml_backend_t backend_from_dev(ggml_backend_dev_t dev) const; + // // graph // @@ -420,6 +431,20 @@ struct llama_context { // host buffer for the model output (logits and embeddings) ggml_backend_buffer_ptr buf_output; + // device buffer with enabled layer inputs, [n_layer_inp*n_embd, n_batch], row = batch index + ggml_context_ptr ctx_layer_inp_dev; + ggml_backend_buffer_ptr buf_layer_inp_dev; + ggml_tensor * layer_inp_dev = nullptr; + ggml_backend_dev_t layer_inp_dev_dev = nullptr; + ggml_backend_t layer_inp_backend = nullptr; + + // ordering with the reader context (no host sync): + // written - recorded by this context after the batch, waited on by the reader + // read - recorded by the reader after it consumed the rows, waited on by this context + ggml_backend_event_ptr ev_layer_inp_written; + ggml_backend_event_ptr ev_layer_inp_read; + bool ev_layer_inp_read_recorded = false; + // keep copies of the per-sequence memory on the device std::map mem_storage; diff --git a/src/llama-cparams.h b/src/llama-cparams.h index c02e8bcac8ab..9141cba3ea93 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -62,6 +62,7 @@ struct llama_cparams { size_t moe_cache_size; std::vector embeddings_layer_inp; // [n_layer()] extract input embeddings for layer + bool embeddings_layer_inp_dev; // write enabled layer inputs to a device buffer instead of host enum llama_context_type ctx_type; enum llama_rope_scaling_type rope_scaling_type; diff --git a/src/llama-ext.h b/src/llama-ext.h index fcd91e912b1e..23a781404bd8 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -164,6 +164,19 @@ LLAMA_API void llama_set_embeddings_layer_inp(struct llama_context * ctx, uint32 // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); LLAMA_API float * llama_get_embeddings_layer_inp(struct llama_context * ctx, uint32_t lid); +// Write the enabled layer inputs to a device buffer [n_layer_inp*n_embd, n_batch] instead of host memory. +// Row r holds the token at batch index r, layers in ascending layer id order. +LLAMA_API void llama_set_embeddings_layer_inp_dev(struct llama_context * ctx, bool value); + +// Whether decode accepts entries added with llama_batch_ext_add_embd_row() +// (needs a DFlash/DSpark decoder context whose ctx_other has a layer input device buffer). +LLAMA_API bool llama_supports_embd_row(const struct llama_context * ctx); + +// Add an entry whose embedding is row `row` of the layer input device buffer of ctx_other. +// All entries of the batch must be of this kind. Positions are set with llama_batch_ext_set_pos(), n_pos_per_embd of them. +// Returns the batch index (>= 0), or the llama_batch_ext_add() error codes. +LLAMA_API int32_t llama_batch_ext_add_embd_row(struct llama_batch_ext * batch, llama_seq_id seq_id, int32_t row); + LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); // diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 725bdb048e51..fe8bdebe5e2c 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -138,6 +138,12 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens)); } + if (ubatch->embd_row && embd_rows) { + const int64_t n_tokens = ubatch->n_tokens; + + ggml_backend_tensor_set(embd_rows, ubatch->embd_row, 0, n_tokens*ggml_element_size(embd_rows)); + } + if (ubatch->embd && embd && !ubatch->is_mixed()) { GGML_ASSERT(n_embd == embd->ne[0]); @@ -197,6 +203,7 @@ bool llm_graph_input_embd::can_reuse(const llm_graph_params & params) { res &= (!mixed_tokens) || mixed_tokens->ne[0] == llm_graph_n_tok_rows(params.ubatch); res &= (!mixed_embd) || mixed_embd->ne[1] == params.ubatch.n_tokens; res &= (!scale_rows) || scale_rows->ne[1] == params.ubatch.n_tokens; + res &= (!params.ubatch.embd_row) || (embd_rows && embd_rows->ne[0] == params.ubatch.n_tokens); return res; } @@ -232,6 +239,22 @@ bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) { return res; } +void llm_graph_input_batch_idxs::set_input(const llama_ubatch * ubatch) { + const int64_t n_tokens = ubatch->n_tokens; + + GGML_ASSERT(ubatch->data && ubatch->data->batch_idxs.size() == (size_t) n_tokens); + GGML_ASSERT(ggml_backend_buffer_is_host(idxs->buffer)); + + int64_t * data = (int64_t *) idxs->data; + for (int64_t i = 0; i < n_tokens; ++i) { + data[i] = ubatch->data->batch_idxs[i]; + } +} + +bool llm_graph_input_batch_idxs::can_reuse(const llm_graph_params & params) { + return idxs->ne[0] == params.ubatch.n_tokens; +} + void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) { if (ubatch->pos && pos) { const int64_t n_tokens = ubatch->n_tokens; @@ -1436,7 +1459,7 @@ void llm_graph_result::set_outputs(const llm_graph_params & params) { } { const auto & embeddings_layer_inp = params.cparams.embeddings_layer_inp; - for (size_t il = 0; il < embeddings_layer_inp.size(); ++il) { + for (size_t il = 0; il < embeddings_layer_inp.size() && !params.cparams.embeddings_layer_inp_dev; ++il) { if (embeddings_layer_inp[il]) { GGML_ASSERT(t_layer_inp[il] != nullptr && "layer input tensor is null"); ggml_set_output(t_layer_inp[il]); @@ -1563,6 +1586,8 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : moe_cache (params.moe_cache), prec_policy (params.prec_policy), hdmd (params.hdmd), + layer_inp_dev (params.layer_inp_dev), + layer_inp_src (params.layer_inp_src), samplers (params.samplers), cb_func (params.cb), res (params.res), @@ -4223,6 +4248,36 @@ void llm_graph_context::build_logits() const { } } +void llm_graph_context::build_layer_inp_dev() const { + if (!cparams.embeddings_layer_inp_dev || layer_inp_dev == nullptr) { + return; + } + + auto inp = std::make_unique(); + + inp->idxs = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_tokens); + ggml_set_input(inp->idxs); + + // row layout: [layer_0 | layer_1 | ...] in ascending layer id order + int64_t col = 0; + for (size_t il = 0; il < cparams.embeddings_layer_inp.size(); ++il) { + if (!cparams.embeddings_layer_inp[il]) { + continue; + } + + ggml_tensor * src = res->t_layer_inp[il]; + GGML_ASSERT(src != nullptr && "layer input tensor is null"); + GGML_ASSERT(src->ne[0] == n_embd); + + ggml_tensor * dst = ggml_view_2d(ctx0, layer_inp_dev, n_embd, layer_inp_dev->ne[1], layer_inp_dev->nb[1], col*n_embd*ggml_element_size(layer_inp_dev)); + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, dst, ggml_reshape_2d(ctx0, src, n_embd, n_tokens), inp->idxs)); + col++; + } + GGML_ASSERT(col*n_embd == layer_inp_dev->ne[0]); + + res->add_input(std::move(inp)); +} + void llm_graph_context::build_sampling() const { if (samplers.empty() || !res->t_logits) { return; diff --git a/src/llama-graph.h b/src/llama-graph.h index b528ee11df46..37b47acbb30a 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -147,6 +147,7 @@ class llm_graph_input_embd : public llm_graph_input_i { ggml_tensor * mixed_slots = nullptr; // I64 [n_tok_rows], mixed path: batch index of the token rows ggml_tensor * mixed_embd = nullptr; // F32 [n_embd, n_batch], mixed path: embd rows, token rows are overwritten ggml_tensor * scale_rows = nullptr; // F32 [1, n_batch], per-row scale: scale_tok for token rows, 1 for embd rows + ggml_tensor * embd_rows = nullptr; // I32 [n_batch], embd row path: rows of ctx_other's layer input device buffer float scale_tok = 1.0f; @@ -171,6 +172,19 @@ class llm_graph_input_embd_h : public llm_graph_input_i { const int64_t n_embd_state = 0; }; +// original batch index of each ubatch token, used as set_rows destination rows +class llm_graph_input_batch_idxs : public llm_graph_input_i { +public: + llm_graph_input_batch_idxs() = default; + virtual ~llm_graph_input_batch_idxs() = default; + + void set_input(const llama_ubatch * ubatch) override; + + bool can_reuse(const llm_graph_params & params) override; + + ggml_tensor * idxs = nullptr; // I64 [n_batch] +}; + class llm_graph_input_pos : public llm_graph_input_i { public: llm_graph_input_pos(uint32_t n_pos_per_embd, llama_rope_section_order rope_section_order = LLAMA_ROPE_SECTION_ORDER_TYXZ) @@ -794,6 +808,9 @@ struct llm_graph_params { const llama_hadamard * hdmd = nullptr; + ggml_tensor * layer_inp_dev = nullptr; // own layer input device buffer (write) + ggml_tensor * layer_inp_src = nullptr; // ctx_other's layer input device buffer (read) + std::map samplers; static bool samplers_equal( @@ -833,7 +850,8 @@ struct llm_graph_params { (!ubatch.embd && !other.ubatch.embd) || (ubatch.token && other.ubatch.token && ubatch.embd && other.ubatch.embd) ) && - (!ubatch.embd_state == !other.ubatch.embd_state); + (!ubatch.embd_state == !other.ubatch.embd_state) && + (!ubatch.embd_row == !other.ubatch.embd_row); // when we split the batch using "equal_seqs" we have to verify that the participating sequences are the same // the reason is because the set of attention streams would be different for different sequences @@ -885,6 +903,8 @@ struct llm_graph_params { cparams.embeddings_nextn == other.cparams.embeddings_nextn && cparams.embeddings_nextn_masked == other.cparams.embeddings_nextn_masked && cparams.causal_attn == other.cparams.causal_attn && + layer_inp_dev == other.layer_inp_dev && + layer_inp_src == other.layer_inp_src && arch == other.arch && gtype == other.gtype && cvec == other.cvec && @@ -1052,6 +1072,9 @@ struct llm_graph_context { const llama_hadamard * hdmd; + ggml_tensor * layer_inp_dev; + ggml_tensor * layer_inp_src; + std::map samplers; const llm_graph_cb & cb_func; @@ -1450,6 +1473,9 @@ struct llm_graph_context { void build_sampling() const; + // write enabled layer inputs into layer_inp_dev at their batch rows + void build_layer_inp_dev() const; + // // dense (out) // diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 10da95ae85cc..d5506f7a7bf2 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -160,6 +160,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) { /*.token =*/ data->token.empty() ? nullptr : data->token.data(), /*.embd =*/ nullptr, /*.embd_state =*/ nullptr, + /*.embd_row =*/ nullptr, /*.pos =*/ data->pos.data(), /*.n_seq_id =*/ data->n_seq_id.data(), /*.seq_id =*/ data->seq_id.data(), diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 52893876dfaf..7529f55c2025 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -3400,6 +3400,8 @@ ggml_cgraph * llama_model::build_graph(const llm_graph_params & params) const { // TODO: move reranking logic here and generalize llm->build_dense_out(dense_2_out_layers, dense_2_out_layers_b, dense_3_out_layers); + llm->build_layer_inp_dev(); + llm->res->set_outputs(params); return llm->res->get_gf(); diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index 209498421cd4..fdd67e01785b 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -434,6 +434,23 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_grap ggml_build_forward_expand(gf, cur); } +// target features of an injection batch: host embd rows, or rows of the target's layer input device buffer +static ggml_tensor * build_inp_target_features(llm_graph_context & g, llm_graph_input_embd & inp, int64_t n_embd_inp) { + if (g.ubatch.embd_row) { + GGML_ASSERT(g.layer_inp_src != nullptr && g.layer_inp_src->ne[0] == n_embd_inp); + + inp.embd_rows = ggml_new_tensor_1d(g.ctx0, GGML_TYPE_I32, g.n_tokens); + ggml_set_input(inp.embd_rows); + + return ggml_get_rows(g.ctx0, g.layer_inp_src, inp.embd_rows); + } + + inp.embd = ggml_new_tensor_2d(g.ctx0, GGML_TYPE_F32, n_embd_inp, g.n_tokens); + ggml_set_input(inp.embd); + + return inp.embd; +} + // DSpark (DFlash + Markov & Confidence head): Markov bias on the draft logits, chained per block position static void build_dspark_markov_head(llm_graph_context & g, const llama_model & model, ggml_tensor * tokens, ggml_tensor * hidden = nullptr) { ggml_context * ctx0 = g.ctx0; @@ -939,13 +956,10 @@ llama_model_dflash::graph::graph(const llama_model & model, const llm_gra // KV cache injection ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1219,13 +1233,10 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ // KV cache injection: fused target features from the encoder ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1418,13 +1429,10 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll }; ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); @@ -1668,13 +1676,10 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model }; ASSERT_EMBD_OR_TOKEN(ubatch); - if (ubatch.embd) { + if (ubatch.embd || ubatch.embd_row) { auto inp = std::make_unique(n_embd_inp); - inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_inp, n_tokens); - ggml_set_input(inp->embd); - - ggml_tensor * inp_target = inp->embd; + ggml_tensor * inp_target = build_inp_target_features(*this, *inp, n_embd_inp); cb(inp_target, "inp_target_features", -1); res->add_input(std::move(inp)); From 51fe42c5d14bfb9adf0612ff98a9ab79e0e7275c Mon Sep 17 00:00:00 2001 From: Mark Todorovich Date: Mon, 2 Nov 2026 21:13:38 -0800 Subject: [PATCH 002/286] hexagon: tiled hvx permute (#31446) * hexagon: add tiled HVX transpose path for permuted CONT/CPY Permuted copies where dst dim 0 is strided in src, but some other src dim is contiguous, used to fall back to HTP_COPY_KERNEL_RESHAPE. That issues one DMA descriptor per element. The channel-first <-> channel-last permutes around per-channel norms (e.g. [W,H,T,C] <-> [C,W,H,T] in video VAEs) ht this path, and these copies ran at about 45 MB/s. Add HTP_COPY_KERNEL_TRANSPOSE for same-type, same-shape f32/f16 copies into a contiguous dst. Host: - drop extent-1 dims and merge dims that stay adjacent in src into a canonical view, stored in the kernel params. There are no minimum size limits. - the copy is treated as a 2D transpose between dst dim 0 and the contiguous src dim, batched over the remaining dims. - tiles are sized to 64 KB per buffer and halved until they fit in session VTCM. blkk is capped so the halfword vgather offsets fit in 16 bits. Kernel, per tile: - 2D DMA of blk0 strided src rows (blkk contiguous elements each) into VTCM - vgather transpose in 32x32 (f32) or 64x64 (f16) blocks - 2D DMA of blkk contiguous rows out to dst Staged and transposed buffers are double-buffered, so the next tile's input DMA overlaps the current transpose and output DMA. Tiles are split across threads and mdev sessions. Small dims: - packed mode: when dim k follows dim 0 and dim 0 divides the HVX lanes (f32: 2/4/8/16, f16: 2/4/.../32), one vgather writes several whole dst rows, and each tile goes out as one contiguous DMA. - when the contiguous src dim fits in one vector, it is staged exactly and the tile is made long along dim 0. All other copies select the same kernels as before. Tests: add test_cont cases for perms {1,2,3,0} and {3,0,1,2}, f32 and f16, at various sizes. * hexagon: cpy transpose cleanups - fix vgather halfword offset comments: offsets are unsigned 16-bit, caps are kept lower for speed - set packed directly in the packed case - add dim_k == 3 CONT tests (full reversal) Assisted-by: Claude Opus 5.5 * hexagon: use cpy transpose path for ne[0] == 1 permutes - drop the nb[0] check in the transpose solver: size-1 dims are skipped, and the canonical tp.nb[0] check covers packed dim 0 - try the transpose path before SAMESHAPE_SAMETYPE, which did one DMA per element for these - add ne[0] == 1 permute CPY tests Assisted-by: Claude Opus 5.5 * hex-cpy: further improve cpy Bring in improvements from aizip branch/PR. Cleanup kparams and replace divs with fastdiv. Co-authored-by: Yiwei Shao * hex-cpy: avoid vtcm bank conflicts * hex-cpy: trim down the comments * cpy-test: trim down new tests that cover permute scenarios --------- Co-authored-by: Max Krasnyansky Co-authored-by: Yiwei Shao --- ggml/src/ggml-hexagon/htp-kparams.cpp | 127 +++++++++++++++ ggml/src/ggml-hexagon/htp/cpy-ops.c | 214 ++++++++++++++++++++++++++ ggml/src/ggml-hexagon/htp/cpy-ops.h | 95 ++++++++++++ tests/test-backend-ops.cpp | 27 ++++ 4 files changed, 463 insertions(+) diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 03b5a3b506d3..e1d3385e2210 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -2660,6 +2660,129 @@ static bool htp_kparams_solve_concat( return false; } +// Permuted copy with strided dim 0 (e.g. channel-first <-> channel-last) +static bool htp_kparams_solve_cpy_transpose( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * src0, + const struct ggml_tensor * dst, + uint32_t type_size, + struct htp_copy_kernel_params * kparams) { + + // View contiguous dst with src shape to support CPY with reshape + const bool dst_cont = ggml_is_contiguous(dst); + if (!dst_cont && !ggml_are_same_shape(src0, dst)) { + return false; + } + + // Drop size-1 dims and merge dims adjacent in both tensors + size_t ne[GGML_MAX_DIMS]; + size_t nbs[GGML_MAX_DIMS]; + size_t nbd[GGML_MAX_DIMS]; + size_t dst_stride = type_size; + int n = 0; + for (int d = 0; d < GGML_MAX_DIMS; d++) { + const size_t sd = dst_cont ? dst_stride : dst->nb[d]; + dst_stride *= (size_t) src0->ne[d]; + if (src0->ne[d] == 1) { + continue; + } + if (n > 0 && src0->nb[d] == nbs[n - 1] * ne[n - 1] && sd == nbd[n - 1] * ne[n - 1]) { + ne[n - 1] *= (size_t) src0->ne[d]; + continue; + } + ne[n] = (size_t) src0->ne[d]; + nbs[n] = src0->nb[d]; + nbd[n] = sd; + n++; + } + + // Fold leading dense dim into element block + size_t es = type_size; + for (int i = 0; i < n; i++) { + if (nbs[i] == type_size && nbd[i] == type_size) { + es *= ne[i]; + for (int j = i; j + 1 < n; j++) { + ne[j] = ne[j + 1]; + nbs[j] = nbs[j + 1]; + nbd[j] = nbd[j + 1]; + } + n--; + break; + } + } + if (es > 128 || (es & (es - 1)) != 0) { + return false; + } + + // Transpose axes: dim a dense in dst, dim k dense in src + int a = -1; + int k = -1; + for (int i = 0; i < n; i++) { + if (a < 0 && nbd[i] == es) { + a = i; + } + if (k < 0 && nbs[i] == es) { + k = i; + } + } + if (a < 0 || k < 0 || a == k) { + return false; + } + + struct htp_copy_transpose_params tp = {}; + tp.elem_size = (uint8_t) es; + tp.ne[0] = (uint32_t) ne[a]; + tp.ne[1] = (uint32_t) ne[k]; + tp.ne[2] = 1; + tp.src_nb[0] = (uint32_t) nbs[a]; + tp.dst_nb[0] = (uint32_t) nbd[k]; + tp.packed = nbd[k] == ne[a] * es && ne[a] * es < 128 && 128 % (ne[a] * es) == 0; + + size_t ne3 = 1; + int m = 0; + for (int i = 0; i < n; i++) { + if (i == a || i == k) { + continue; + } + if (m == 0) { + tp.ne[2] = (uint32_t) ne[i]; + } else { + ne3 = ne[i]; + } + tp.src_nb[1 + m] = (uint32_t) nbs[i]; + tp.dst_nb[1 + m] = (uint32_t) nbd[i]; + m++; + } + + const uint32_t n_threads = htp_session_n_threads(sess); + const size_t vtcm_size = htp_session_vtcm_size(sess); + uint64_t vtcm = 0; + for (uint32_t blk_bytes = HTP_COPY_TRANSPOSE_BLK_BYTES; ; blk_bytes /= 2) { + if (!htp_copy_transpose_layout_build(&tp, blk_bytes)) { + return false; + } + vtcm = (uint64_t) n_threads * 2 * (tp.in_buf_size + tp.out_buf_size); + if (vtcm <= vtcm_size) { + break; + } + } + + const uint64_t total_tiles = (uint64_t) tp.n0b * tp.nkb * tp.ne[2] * ne3; + if (total_tiles > UINT32_MAX) { + return false; + } + tp.total_tiles = (uint32_t) total_tiles; + tp.div_nkb = init_fastdiv_values(tp.nkb); + tp.div_n0b = init_fastdiv_values(tp.n0b); + tp.div_ne2 = init_fastdiv_values(tp.ne[2]); + + kparams->kernel_type = HTP_COPY_KERNEL_TRANSPOSE; + kparams->n_threads = (uint8_t) n_threads; + kparams->vtcm_size = (uint32_t) vtcm; + kparams->u.transpose = tp; + return true; +} + static bool htp_kparams_solve_cpy( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, @@ -2757,6 +2880,10 @@ static bool htp_kparams_solve_cpy( } } + if (htp_kparams_solve_cpy_transpose(sess, src0, dst, src_type_size, kparams)) { + return true; + } + if (sameshape) { kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE; kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]); diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.c b/ggml/src/ggml-hexagon/htp/cpy-ops.c index fce45c2d4736..59f406c7af8a 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.c +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.c @@ -35,6 +35,10 @@ struct htp_copy_context { struct htp_copy_convert_vtcm_layout layout; uint8_t * vtcm_src0; uint8_t * vtcm_dst; + + uint32_t tile_start; + uint32_t ntiles; + uint32_t tiles_per_thread; }; #define cpy_preamble \ @@ -162,6 +166,171 @@ static inline void cpy_cursor_advance(struct cpy_row_cursor * c, uint32_t n, uin } } +struct cpy_tile_cursor { + uint32_t ibk; + uint32_t ib0; + uint32_t i2; + uint32_t i3; +}; + +static inline void cpy_tile_cursor_advance(struct cpy_tile_cursor * c, const struct htp_copy_transpose_params * tp) { + if (++c->ibk == tp->nkb) { + c->ibk = 0; + if (++c->ib0 == tp->n0b) { + c->ib0 = 0; + if (++c->i2 == tp->ne[2]) { + c->i2 = 0; + c->i3++; + } + } + } +} + +// Gather offsets for element e = l >> log2_m at (e % r, e / r) +static inline HVX_Vector cpy_transpose_offsets(const struct htp_copy_transpose_params * tp) { + const uint32_t r_mask = (1u << tp->log2_r) - 1; + if (tp->elem_size != 2) { + const uint32_t m_mask = (1u << tp->log2_m) - 1; + int32_t offsets[32] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 32; l++) { + const uint32_t e = l >> tp->log2_m; + offsets[l] = (e & r_mask) * tp->in_stride + (e >> tp->log2_r) * tp->elem_size + (l & m_mask) * 4; + } + return *(HVX_Vector *) offsets; + } + int16_t offsets[64] __attribute__((aligned(128))); + for (uint32_t l = 0; l < 64; l++) { + offsets[l] = (l & r_mask) * tp->in_stride + (l >> tp->log2_r) * 2; + } + return *(HVX_Vector *) offsets; +} + +static inline void cpy_transpose_push_in(dma_queue * q, const struct htp_copy_transpose_params * tp, dma_addr_t src, + const struct cpy_tile_cursor * c, uint8_t * buf) { + const uint32_t a0 = c->ib0 * tp->blk0; + const uint32_t ak = c->ibk * tp->blkk; + const uint32_t off = a0 * tp->src_nb[0] + ak * tp->elem_size + c->i2 * tp->src_nb[1] + c->i3 * tp->src_nb[2]; + dma_queue_push(q, dma_make_data(buf, src + off), tp->in_stride, tp->src_nb[0], + MIN(tp->blkk, tp->ne[1] - ak) * tp->elem_size, MIN(tp->blk0, tp->ne[0] - a0)); +} + +// Pipelined tile worker: DMA in -> vgather transpose -> DMA out +static void cpy_thread_transpose(unsigned int nth, unsigned int ith, void * data) { + struct htp_copy_context * ct = (struct htp_copy_context *) data; + struct htp_ops_context * octx = ct->octx; + const struct htp_copy_transpose_params * tp = &ct->kparams->u.transpose; + + const uint32_t t0 = ct->tile_start + ith * ct->tiles_per_thread; + const uint32_t t1 = MIN(t0 + ct->tiles_per_thread, ct->tile_start + ct->ntiles); + if (t0 >= t1) { + return; + } + + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const dma_addr_t src = octx->src[0]->data; + const dma_addr_t dst = octx->dst->data; + + const uint32_t es = tp->elem_size; + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t in_stride = tp->in_stride; + const uint32_t out_stride = tp->out_stride; + + const HVX_Vector offsets = cpy_transpose_offsets(tp); + const uint32_t packed_mu = (ne0 - 1) * in_stride + 127; + const uint32_t packed_step = 128 >> tp->log2_r; + const uint32_t epv = 1u << tp->log2_r; + + uint8_t * in_buf[2]; + uint8_t * out_buf[2]; + in_buf[0] = ct->vtcm_src0 + ith * 2 * tp->in_buf_size; + in_buf[1] = in_buf[0] + tp->in_buf_size; + out_buf[0] = ct->vtcm_dst + ith * 2 * tp->out_buf_size; + out_buf[1] = out_buf[0] + tp->out_buf_size; + + struct cpy_tile_cursor cur; + { + const uint32_t q0 = fastdiv(t0, &tp->div_nkb); + const uint32_t q1 = fastdiv(q0, &tp->div_n0b); + cur.ibk = t0 - q0 * tp->nkb; + cur.ib0 = q0 - q1 * tp->n0b; + cur.i3 = fastdiv(q1, &tp->div_ne2); + cur.i2 = q1 - cur.i3 * tp->ne[2]; + } + + struct cpy_tile_cursor nxt = cur; + + cpy_transpose_push_in(dma_q, tp, src, &cur, in_buf[0]); + + for (uint32_t t = t0, i = 0; t < t1; t++, i++) { + while (dma_queue_depth(dma_q) > (i > 0 ? 1u : 0u)) { + dma_queue_pop(dma_q); + } + + uint8_t * in = in_buf[i & 1]; + uint8_t * out = out_buf[i & 1]; + + const uint32_t a0 = cur.ib0 * tp->blk0; + const uint32_t ak = cur.ibk * tp->blkk; + const uint32_t n0 = MIN(tp->blk0, ne0 - a0); + const uint32_t nk = MIN(tp->blkk, nek - ak); + + if (t + 1 < t1) { + cpy_tile_cursor_advance(&nxt, tp); + cpy_transpose_push_in(dma_q, tp, src, &nxt, in_buf[(i + 1) & 1]); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + if (tp->packed) { + const uint32_t nvec = (nk * ne0 * es + 127) >> 7; + if (es != 2) { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVw((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t v = 0; v < nvec; v++) { + Q6_vgather_ARMVh((HVX_Vector *) (out + v * 128), (size_t) (in + v * packed_step), packed_mu, offsets); + } + } + } else { + for (uint32_t rb = 0; rb < n0; rb += epv) { + const uint32_t mu = (MIN(epv, n0 - rb) - 1) * in_stride + es - 1; + uint8_t * d = out + rb * es; + const uint8_t * s = in + rb * in_stride; + if (es != 2) { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVw((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * es), mu, offsets); + } + } else { + #pragma unroll(4) + for (uint32_t ik = 0; ik < nk; ik++) { + Q6_vgather_ARMVh((HVX_Vector *) (d + ik * out_stride), (size_t) (s + ik * 2), mu, offsets); + } + } + } + } + hvx_gather_sync(out); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) t); + + const uint32_t doff = a0 * es + ak * tp->dst_nb[0] + cur.i2 * tp->dst_nb[1] + cur.i3 * tp->dst_nb[2]; + if (tp->packed) { + const uint32_t n = nk * ne0 * es; + dma_queue_push(dma_q, dma_make_data(dst + doff, out), n, n, n, 1); + } else { + dma_queue_push(dma_q, dma_make_data(dst + doff, out), tp->dst_nb[0], out_stride, n0 * es, nk); + } + + cur = nxt; + } + + dma_queue_flush(dma_q); +} + #define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \ static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \ struct htp_copy_context * ct = (struct htp_copy_context *) data; \ @@ -439,6 +608,48 @@ static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kern return HTP_STATUS_OK; } +static int cpy_transpose(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) { + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + const struct htp_copy_transpose_params * tp = &kparams->u.transpose; + + uint32_t tile_start = 0; + uint32_t ntiles = tp->total_tiles; + + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition( + tp->total_tiles, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + tile_start = range.start; + ntiles = range.count; + } + + if (ntiles == 0) { + return HTP_STATUS_OK; + } + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const uint32_t n_threads = octx->n_threads; + + struct htp_copy_context ct; + ct.octx = octx; + ct.kparams = kparams; + ct.tile_start = tile_start; + ct.ntiles = ntiles; + ct.tiles_per_thread = fastdiv(ntiles + n_threads - 1, &octx->n_threads_div); + + uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + ct.vtcm_src0 = vtcm_base; + ct.vtcm_dst = vtcm_base + (size_t) n_threads * 2 * tp->in_buf_size; + + work_queue_run(octx->ctx->work_queue, cpy_thread_transpose, &ct, n_threads); + return HTP_STATUS_OK; +} + int op_cpy(struct htp_ops_context * octx) { const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params; int status = HTP_STATUS_OK; @@ -459,6 +670,9 @@ int op_cpy(struct htp_ops_context * octx) { case HTP_COPY_KERNEL_RESHAPE: status = cpy_reshape(octx, kparams); break; + case HTP_COPY_KERNEL_TRANSPOSE: + status = cpy_transpose(octx, kparams); + break; case HTP_COPY_KERNEL_GATHER_ROWS: status = htp_gather_rows(octx, octx->src[0], NULL, &kparams->u.gather, kparams->n_threads); break; diff --git a/ggml/src/ggml-hexagon/htp/cpy-ops.h b/ggml/src/ggml-hexagon/htp/cpy-ops.h index c3b97c0cab4b..6bb96ca44758 100644 --- a/ggml/src/ggml-hexagon/htp/cpy-ops.h +++ b/ggml/src/ggml-hexagon/htp/cpy-ops.h @@ -15,6 +15,7 @@ enum htp_copy_kernel_type { HTP_COPY_KERNEL_RESHAPE = 4, HTP_COPY_KERNEL_SCALAR = 5, HTP_COPY_KERNEL_GATHER_ROWS = 6, + HTP_COPY_KERNEL_TRANSPOSE = 7, }; struct htp_copy_convert_params { @@ -32,6 +33,29 @@ struct htp_copy_reshape_params { struct fastdiv_values div_ne02_ne01_ne00; }; +// Batched 2D transpose of elem_size elements (dim 0 dense in dst, dim 1 dense in src) +struct htp_copy_transpose_params { + uint32_t ne[3]; // dims 0..2 + uint32_t src_nb[3]; // src strides for dims 0, 2, 3 + uint32_t dst_nb[3]; // dst strides for dims 1, 2, 3 + struct fastdiv_values div_nkb; + struct fastdiv_values div_n0b; + struct fastdiv_values div_ne2; + uint32_t blk0; // dim-0 elements per tile + uint32_t blkk; // dim-1 elements per tile + uint32_t n0b; // tiles along dim 0 + uint32_t nkb; // tiles along dim 1 + uint32_t in_stride; // VTCM row pitch of staged src tile + uint32_t out_stride; // VTCM row pitch of transposed tile + uint32_t in_buf_size; + uint32_t out_buf_size; + uint32_t total_tiles; + uint8_t elem_size; + uint8_t packed; // gather straight to dst order + uint8_t log2_m; // log2(gather lanes per element) + uint8_t log2_r; // log2(dim-0 elements per vector) +}; + struct htp_copy_kernel_params { uint8_t kernel_type; uint8_t src0_type_size; @@ -46,6 +70,7 @@ struct htp_copy_kernel_params { struct htp_copy_convert_params convert; struct htp_copy_reshape_params reshape; struct htp_gather_rows_params gather; + struct htp_copy_transpose_params transpose; } u; }; @@ -116,6 +141,76 @@ static inline void htp_copy_convert_solve_layout( htp_copy_convert_vtcm_layout_build(layout, ne00, src_type_size, dst_type_size, blk_rows, n_threads); } +#define HTP_COPY_TRANSPOSE_BLK_BYTES (64 * 1024) + +static inline bool htp_copy_transpose_layout_build(struct htp_copy_transpose_params * tp, uint32_t blk_bytes) { + const uint32_t ne0 = tp->ne[0]; + const uint32_t nek = tp->ne[1]; + const uint32_t es = tp->elem_size; + const uint32_t epv = 128 / es; + + uint32_t blk0; + uint32_t blkk; + if (tp->packed) { + // Packed: vector holds epv / ne0 whole dst rows + blk0 = ne0; + blkk = blk_bytes / (ne0 * es); + if (es == 2 && blkk > 16384 / ne0 - 64) { + blkk = 16384 / ne0 - 64; + } + if (blkk > nek) { + blkk = nek; + } + if (blkk == 0) { + return false; + } + } else if (nek <= epv) { + // Stage short contiguous dim and expand dim 0 + blkk = nek; + blk0 = blk_bytes / (nek * es); + blk0 -= blk0 % epv; + if (blk0 > hex_round_up(ne0, epv)) { + blk0 = hex_round_up(ne0, epv); + } + if (blk0 < epv) { + return false; + } + } else { + blk0 = hex_round_up(ne0, epv); + if (blk0 > 256) { + blk0 = 256; + } + blkk = blk_bytes / (blk0 * es); + blkk -= blkk % epv; + // Cap f16 blkk to 256 for 16-bit gather offsets + const uint32_t max_blkk = (es == 2 ? 512 : 4096) / es; + if (blkk > max_blkk) { + blkk = max_blkk; + } + if (blkk > hex_round_up(nek, epv)) { + blkk = hex_round_up(nek, epv); + } + if (blkk < epv) { + return false; + } + } + + tp->blk0 = blk0; + tp->blkk = blkk; + tp->n0b = (ne0 + blk0 - 1) / blk0; + tp->nkb = (nek + blkk - 1) / blkk; + // Pad row pitch by 1 element to avoid VTCM bank conflicts + const uint32_t pad = (es == 2) ? 2 : 4; + tp->in_stride = blkk * es + pad; + tp->out_stride = blk0 * es; + // Extra vector for packed gather overhang + tp->in_buf_size = hex_round_up(blk0 * tp->in_stride + (tp->packed ? 128 : 0), 128); + tp->out_buf_size = hex_round_up(blkk * tp->out_stride, 128); + tp->log2_m = es == 2 ? 0 : __builtin_ctz(es) - 2; + tp->log2_r = __builtin_ctz(tp->packed ? ne0 : epv); + return true; +} + #if defined(__cplusplus) static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob"); #else diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index eec0423cbef5..64fdb8259a30 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10865,6 +10865,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cpy(GGML_TYPE_I32, GGML_TYPE_I32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {256, 1, 4, 1}, {-1,-1,-1,-1}, {1, 2, 0, 3}, {0, 0, 0, 0})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2097121, 1, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {1, 64, 48, 1}, {-1,-1,-1,-1}, {0, 2, 1, 3})); // ne[0] == 1 permute test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {2, 2, 524281, 1}, {-1,-1,-1,-1}, {1, 0, 2, 3})); test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {128, 2, 3, 1}, {128, 2, 3, 1}, {0, 0, 0, 0}, {0, 0, 0, 0}, false, {128, 4, 3, 1})); // strided dst @@ -10931,6 +10933,31 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {512, 8193, 2, 1}, false, {2, 1, 0, 3})); test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {256, 16385, 2, 1}, false, {0, 2, 1, 3})); + // channel swap with sizes not multiples of 32/64 tile + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + } + + // small channel dims and extent-1 dims + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {8, 104, 60, 13}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {16, 1, 50, 7}, false, {1, 2, 3, 0})); + + // full reversal: contiguous src dim lands on dst dim 3 + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 5, 7, 9}, false, {3, 2, 1, 0})); + test_cases.emplace_back(new test_cont(GGML_TYPE_F32, {33, 64, 32, 16}, false, {3, 2, 1, 0})); + + // permutes that move dense blocks (ne[0] stays dim 0) + for (ggml_type type_dst : { GGML_TYPE_F32, GGML_TYPE_F16 }) { + test_cases.emplace_back(new test_cont(type_dst, {4, 65, 9, 2}, false, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_cont(GGML_TYPE_F16, {64, 9, 7, 1}, false, {0, 2, 1, 3})); + + // transpose with i32 and CPY with reshape + test_cases.emplace_back(new test_cont(GGML_TYPE_I32, {33, 17, 5, 19}, false, {1, 2, 3, 0})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F32, GGML_TYPE_F32, {33, 17, 5, 1}, {17, 33, 5, 1}, {0, 0, 0, 0}, {1, 0, 2, 3})); + test_cases.emplace_back(new test_cpy(GGML_TYPE_F16, GGML_TYPE_F16, {8, 52, 39, 1}, {8, 39, 52, 1}, {0, 0, 0, 0}, {0, 2, 1, 3})); + auto add_test_bin_bcast = [&](ggml_type type, std::array ne, std::array nr, bool perm1 = false, bool src_overlap = false) { for (auto op : {ggml_add, ggml_sub, ggml_mul, ggml_div}) { test_cases.emplace_back(new test_bin_bcast(op, type, ne, nr, 1, perm1, src_overlap)); From ee17f293b638295651c6ff4752a4dce102c6b8e4 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 13:51:53 +0800 Subject: [PATCH 003/286] kv-cache : never return 0 kv_rows for sequences without cells (#31570) A cache that shares the cells of another one (f.ex. the gemma4 assistant) can build a graph before its sequences have any cells, f.ex. at the server warmup. get_n_kv_rows() then returned 0, which means that kv_rows are not used, so the graph fell back to the dense KQ mask and the scheduler re-planned for it. Later graphs with larger masks then aborted with GGML_SCHED_DEBUG_REALLOC=1. --- src/llama-kv-cache.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index 84f547ae18d6..d9f46f0adccf 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1413,7 +1413,8 @@ uint32_t llama_kv_cache::get_n_kv_rows(const llama_ubatch & ubatch) const { } // pad like n_kv so that the graph can be reused - return std::min(GGML_PAD(n_max, n_pad_cur), get_size()); + // never 0, it means that kv_rows are not used (f.ex. the sequences have no cells yet in a cache that shares the cells of another one) + return std::min(std::max(n_pad_cur, GGML_PAD(n_max, n_pad_cur)), get_size()); } ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const { From 713319e28492d319b81e49646151f6bbdb9d4551 Mon Sep 17 00:00:00 2001 From: Leon <118652405+LeON-Nie-code@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:54:00 +0800 Subject: [PATCH 004/286] webgpu: add ARANGE support (#31455) --- docs/ops.md | 2 +- docs/ops/WebGPU.csv | 12 +++++-- .../ggml-webgpu/ggml-webgpu-shader-lib.hpp | 19 ++++++++++++ ggml/src/ggml-webgpu/ggml-webgpu.cpp | 31 +++++++++++++++++++ ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl | 22 +++++++++++++ tests/test-backend-ops.cpp | 9 ++++++ 6 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl diff --git a/docs/ops.md b/docs/ops.md index 2885441c5a37..3e14b06cb16a 100644 --- a/docs/ops.md +++ b/docs/ops.md @@ -19,7 +19,7 @@ Legend: | ADD | ❌ | ✅ | ✅ | ✅ | 🟡 | 🟡 | 🟡 | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | ADD1 | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | | ADD_ID | ❌ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | -| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | +| ARANGE | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGMAX | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | | ARGSORT | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | | CEIL | ❌ | ❌ | ✅ | 🟡 | 🟡 | 🟡 | ✅ | 🟡 | ✅ | ✅ | ✅ | ❌ | ❌ | diff --git a/docs/ops/WebGPU.csv b/docs/ops/WebGPU.csv index c19396c03e4d..d402092c4186 100644 --- a/docs/ops/WebGPU.csv +++ b/docs/ops/WebGPU.csv @@ -12326,8 +12326,16 @@ "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[512,34,2,1],pad_0=10,pad_1=9","support","0","no","WebGPU" "WebGPU: WebGPU","PAD_REFLECT_1D","type=f32,ne_a=[3000,384,4,1],pad_0=10,pad_1=9","support","0","no","WebGPU" "WebGPU: WebGPU","ROLL","shift0=3,shift1=-2,shift3=1,shift4=-1","support","0","no","WebGPU" -"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","0","no","WebGPU" -"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","0","no","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=10.000000,step=1.000000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=3.000000,stop=3.500000,step=1.000000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-10.000000,stop=10.000000,step=0.300000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=62.750000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.000000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=63.250000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=254.750000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.000000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=-1.000000,stop=255.250000,step=0.250000","support","1","yes","WebGPU" +"WebGPU: WebGPU","ARANGE","type=f32,start=0.000000,stop=1048576.000000,step=1.000000","support","1","yes","WebGPU" "WebGPU: WebGPU","TIMESTEP_EMBEDDING","type=f32,ne_a=[2,1,1,1],dim=320,max_period=10000","support","0","no","WebGPU" "WebGPU: WebGPU","LEAKY_RELU","type=f32,ne_a=[10,5,4,3],negative_slope=0.100000","support","0","no","WebGPU" "WebGPU: WebGPU","CUMSUM","type=f32,ne=[10,5,4,3]","support","1","yes","WebGPU" diff --git a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp index 1e63a9fac052..d1cd0818ebc5 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu-shader-lib.hpp @@ -1233,6 +1233,7 @@ class ggml_webgpu_shader_lib { std::unordered_map argsort_pipelines; // key is order std::unordered_map argsort_merge_pipelines; // key is order std::unordered_map cumsum_pipelines; // key is fixed, no variants yet + std::unordered_map arange_pipelines; // key is fixed, no variants yet std::unordered_map row_norm_pipelines; // op/inplace @@ -1943,6 +1944,24 @@ class ggml_webgpu_shader_lib { return gated_delta_net_pipelines[key]; } + webgpu_pipeline get_arange_pipeline(const ggml_webgpu_shader_lib_context & context) { + auto it = arange_pipelines.find(1); + if (it != arange_pipelines.end()) { + return it->second; + } + + std::vector defines; + defines.push_back(std::string("WG_SIZE=") + std::to_string(context.max_wg_size)); + + auto processed = preprocessor.preprocess(wgsl_arange, defines); + auto decisions = std::make_shared(); + decisions->wg_size = context.max_wg_size; + webgpu_pipeline pipeline = ggml_webgpu_create_pipeline(device, processed, "arange"); + pipeline.context = decisions; + arange_pipelines[1] = pipeline; + return arange_pipelines[1]; + } + webgpu_pipeline get_pad_pipeline(const ggml_webgpu_shader_lib_context & context) { ggml_webgpu_pad_pipeline_key key = {}; key.circular = ggml_get_op_params_i32(context.dst, 8) != 0; diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index 44ff0b10cd99..bb2deb9da3f0 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -856,6 +856,32 @@ static webgpu_encoded_op ggml_webgpu_set(webgpu_context & ctx, return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); } +static webgpu_encoded_op ggml_webgpu_arange(webgpu_context & ctx, ggml_tensor * dst) { + ggml_webgpu_shader_lib_context shader_lib_ctx = {}; + shader_lib_ctx.max_wg_size = ctx->global_ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + webgpu_pipeline pipeline = ctx->shader_lib->get_arange_pipeline(shader_lib_ctx); + auto * decisions = static_cast(pipeline.context.get()); + + const uint32_t ne = (uint32_t) ggml_nelements(dst); + std::vector params = { + ne, + (uint32_t) (ggml_webgpu_tensor_misalignment(ctx, dst) / ggml_type_size(dst->type)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 0)), + ggml_webgpu_u32_from_f32(ggml_get_op_params_f32(dst, 2)), + }; + + std::vector entries = { + ggml_webgpu_make_tensor_bind_group_entry(ctx, 0, dst), + }; + + uint32_t wg_x; + uint32_t wg_y; + uint32_t total_wg = CEIL_DIV(ne, decisions->wg_size); + compute_2d_workgroups(total_wg, ctx->global_ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + return ggml_backend_webgpu_build(ctx, pipeline, params, entries, wg_x, wg_y); +} + static webgpu_encoded_op ggml_webgpu_pad(webgpu_context & ctx, ggml_tensor * src, ggml_tensor * dst) { ggml_webgpu_shader_lib_context shader_lib_ctx = {}; shader_lib_ctx.src0 = src; @@ -3480,6 +3506,8 @@ static std::optional ggml_webgpu_encode(webgpu_context ctx, return ggml_webgpu_gated_delta_net(ctx, src0, src1, src2, node->src[3], node->src[4], node->src[5], node, nullptr); } + case GGML_OP_ARANGE: + return ggml_webgpu_arange(ctx, node); case GGML_OP_PAD: return ggml_webgpu_pad(ctx, src0, node); case GGML_OP_ARGMAX: @@ -4784,6 +4812,9 @@ static bool ggml_backend_webgpu_device_supports_op(ggml_backend_dev_t dev, const case GGML_OP_COS: supports_op = (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16) && (src0->type == op->type); break; + case GGML_OP_ARANGE: + supports_op = op->type == GGML_TYPE_F32 && ggml_is_contiguous(op); + break; case GGML_OP_PAD: supports_op = op->type == GGML_TYPE_F32 && src0->type == GGML_TYPE_F32; break; diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl new file mode 100644 index 000000000000..707a66536222 --- /dev/null +++ b/ggml/src/ggml-webgpu/wgsl-shaders/arange.wgsl @@ -0,0 +1,22 @@ +@group(0) @binding(0) +var dst: array; + +struct Params { + ne: u32, + offset_dst: u32, + start: f32, + step: f32, +}; + +@group(0) @binding(1) +var params: Params; + +@compute @workgroup_size(WG_SIZE) +fn main(@builtin(num_workgroups) num_wg: vec3, @builtin(global_invocation_id) gid: vec3) { + let i = gid.x + (num_wg.x * u32(WG_SIZE)) * gid.y; + if (i >= params.ne) { + return; + } + + dst[params.offset_dst + i] = params.start + params.step * f32(i); +} diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 64fdb8259a30..68b25c876484 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -12206,6 +12206,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_roll()); test_cases.emplace_back(new test_roll(3, -2, 1, -1, true)); test_cases.emplace_back(new test_arange()); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 3.0f, 3.5f, 1.0f)); + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -10.0f, 10.0f, 0.3f)); + for (float n : {255.0f, 256.0f, 257.0f, 1023.0f, 1024.0f, 1025.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, -1.0f, -1.0f + n * 0.25f, 0.25f)); + } test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, 1048576.0f, 1.0f)); test_cases.emplace_back(new test_timestep_embedding()); test_cases.emplace_back(new test_leaky_relu()); @@ -12759,6 +12764,10 @@ static std::vector> make_test_cases_eval() { static std::vector> make_test_cases_perf() { std::vector> test_cases; + for (float n : {128.0f, 4096.0f, 32768.0f, 1048576.0f}) { + test_cases.emplace_back(new test_arange(GGML_TYPE_F32, 0.0f, n, 1.0f)); + } + // delta-net conv input: conv state followed by the transposed qkv of the ubatch test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16)); From 74101b3cda9c528e5e829f5e508b62061f1cdb8e Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 13:54:47 +0800 Subject: [PATCH 005/286] ci : update the HIP VGPR spill ignore list for the kv_rows FA kernel argument (#31550) The kv_rows argument added to the FA kernels in #29510 changes their mangled names, so the known spills were no longer ignored. --- scripts/hip/gcn-cdna-vgpr-check.py | 42 +++++++++++++++--------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 146bc8ff58e4..a09891fdc3a5 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -64,27 +64,27 @@ def main(): '_ZL12rwkv_wkv_f32ILi128EEviiiiPKfS1_S1_S1_S1_S1_Pf', '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', '_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiS4_PfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', } functions = parse_log_file(log_file) From 4402e65401f11587e1ed2e549b2c0c0285d871ee Mon Sep 17 00:00:00 2001 From: frogscooper Date: Mon, 2 Nov 2026 21:55:11 -0800 Subject: [PATCH 006/286] chat : allow constraint type after gpt-oss final channel (#31556) --- common/parsers/gpt-oss.cpp | 5 ++--- tests/test-chat.cpp | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cpp index 5ccdff57de5a..4cbf929f9bfd 100644 --- a/common/parsers/gpt-oss.cpp +++ b/common/parsers/gpt-oss.cpp @@ -71,6 +71,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto content = p.rule("message-content", p.until("<|end|>")); auto channel = p.literal("<|channel|>") + (p.literal("commentary") | p.literal("analysis")); auto constrain_type = p.chars("[A-Za-z0-9_-]", 1, -1); + auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); // Occasionally, gpt-oss-20b will prefix channels with this commentary auto stray_commentary = p.optional(p.literal("<|channel|>commentary") + p.optional(p.literal(" to=assistant"))); @@ -84,7 +85,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto analysis = p.ref("analysis"); auto preamble = p.rule("preamble", p.literal("<|channel|>commentary<|message|>") + p.content(content) + end); - auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final<|message|>") + p.content(content)); + auto final_msg = p.rule("final", stray_commentary + p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(content)); // Consume any unsolicited tool calls, e.g. builtin functions auto unsolicited = p.rule("unsolicited", p.atomic(p.optional(channel) + p.literal(" to=") + content + end)); @@ -92,7 +93,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & auto any = p.rule("any", preamble | analysis); if (has_response_format) { - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); auto response_format = p.rule("response-format", p.literal("<|channel|>final") + constraint + p.literal("<|message|>") + p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); @@ -109,7 +109,6 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & const auto params = common_chat_tool_parameters(function); auto func_name = p.literal(" to=functions.") + p.tool_name(p.literal(name)); - auto constraint = p.optional(p.space() + p.optional(p.literal("<|constrain|>")) + constrain_type); auto args = p.tool_args(p.schema(p.json(), "tool-" + std::to_string(tool_index) + "-schema", params)); // recipient in role header diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index fa4525389976..4d10815e5fc7 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -6937,6 +6937,10 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // Basic content only - final channel tst.test("<|channel|>final<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + // Final channel with a constraint type, with and without <|constrain|> (#31520) + tst.test("<|channel|>final code<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + tst.test("<|channel|>final <|constrain|>json<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); + // Basic content only - commentary channel tst.test("<|channel|>commentary<|message|>Hello, world!\nWhat's up?").expect(message_assist).run(); From e6192de0a62b73f566f825ecb531f7e4fe699eae Mon Sep 17 00:00:00 2001 From: shaofeiqi Date: Mon, 2 Nov 2026 21:58:19 -0800 Subject: [PATCH 007/286] opencl: add initial iq4_xs support (#31339) * opencl: add iq4_xs AOS mv Assisted-by: Codex * opencl: add iq4_xs SOA mv and tiled mm Assisted-by: Codex * opencl: add iq4_xs noshuffle gemm/gemv for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: add iq4_xs dp4a for adreno Co-authored-by: Shaofei Qi Assisted-by: Codex * opencl: optimize iq4_xs adreno decode and narrow batches Co-authored-by: Shaofei Qi Assisted-by: Codex --------- Co-authored-by: Li He --- ggml/src/ggml-opencl/CMakeLists.txt | 7 + ggml/src/ggml-opencl/ggml-opencl.cpp | 623 +++++++++++++++++- ggml/src/ggml-opencl/kernels/cvt.cl | 95 +++ .../kernels/gemm_noshuffle_iq4_xs_f32.cl | 88 +++ .../gemm_noshuffle_iq4_xs_q8_1_dp4a.cl | 135 ++++ .../kernels/gemv_noshuffle_iq4_xs_f32.cl | 85 +++ ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl | 10 + .../kernels/mul_mm_iq4_xs_f32_l4_lm.cl | 186 ++++++ .../ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl | 134 ++++ .../kernels/mul_mv_iq4_xs_f32_flat.cl | 134 ++++ 10 files changed, 1496 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl create mode 100644 ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl create mode 100644 ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl diff --git a/ggml/src/ggml-opencl/CMakeLists.txt b/ggml/src/ggml-opencl/CMakeLists.txt index 79fc37b60990..20dd595c7e75 100644 --- a/ggml/src/ggml-opencl/CMakeLists.txt +++ b/ggml/src/ggml-opencl/CMakeLists.txt @@ -131,6 +131,9 @@ set(GGML_OPENCL_KERNELS gemm_noshuffle_iq3_xxs_f32 gemm_noshuffle_iq3_xxs_q8_1_dp4a gemv_noshuffle_iq3_xxs_f32 + gemm_noshuffle_iq4_xs_f32 + gemm_noshuffle_iq4_xs_q8_1_dp4a + gemv_noshuffle_iq4_xs_f32 gemm_noshuffle_q3_k_q8_1_dp4a gemv_noshuffle_q3_k_f32 mul_mv_q2_k_f32 @@ -165,6 +168,10 @@ set(GGML_OPENCL_KERNELS mul_mv_iq3_s_f32 mul_mv_iq3_s_f32_flat mul_mm_iq3_s_f32_l4_lm + iq4_xs_grid + mul_mv_iq4_xs_f32 + mul_mv_iq4_xs_f32_flat + mul_mm_iq4_xs_f32_l4_lm mul_mv_q3_k_f32_flat mul_mm_q3_k_f32_l4_lm mul_mv_q2_k_f32_flat diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index 9999aac905ad..f0d401d55d22 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -1074,6 +1074,16 @@ struct ggml_backend_opencl_context { ggml_cl_buffer iq3_s_grid_buffer; cl_mem iq3_s_grid_image = nullptr; cl_kernel kernel_mul_mv_iq3_s_f32; + cl_kernel kernel_convert_block_iq4_xs, kernel_restore_block_iq4_xs; + cl_kernel kernel_convert_block_iq4_xs_noshuffle = nullptr, kernel_restore_block_iq4_xs_noshuffle = nullptr; + cl_kernel kernel_mul_mv_iq4_xs_f32_flat; + cl_kernel kernel_mul_mm_iq4_xs_f32_l4_lm; + cl_kernel kernel_gemv_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_f32 = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = nullptr; + cl_kernel kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = nullptr; + int iq4_xs_mv_nsg = 8; + cl_kernel kernel_mul_mv_iq4_xs_f32; cl_kernel kernel_mul_mv_q4_K_f32; cl_kernel kernel_mul_mv_q4_K_f32_flat; cl_kernel kernel_mul_mv_q5_K_f32; @@ -2163,6 +2173,10 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { CL_CHECK((backend_ctx->kernel_restore_block_iq3_xxs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_xxs", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq3_s", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_iq3_s = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq3_s", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs_noshuffle = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs_noshuffle", &err), err)); + CL_CHECK((backend_ctx->kernel_convert_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_iq4_xs", &err), err)); + CL_CHECK((backend_ctx->kernel_restore_block_iq4_xs = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_iq4_xs", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_restore_block_q3_K = clCreateKernel(backend_ctx->program_cvt, "kernel_restore_block_q3_K", &err), err)); CL_CHECK((backend_ctx->kernel_convert_block_q2_K = clCreateKernel(backend_ctx->program_cvt, "kernel_convert_block_q2_K", &err), err)); @@ -3054,6 +3068,59 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string iq4_xs_grid_src { + #include "iq4_xs_grid.cl.h" + }; +#else + const std::string iq4_xs_grid_src = read_file("iq4_xs_grid.cl"); +#endif + + // mul_mv_iq4_xs_f32_flat + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_xs_f32_flat.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32_flat.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32_flat = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32_flat", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mm_iq4_xs_f32_l4_lm + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mm_iq4_xs_f32_l4_lm.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mm_iq4_xs_f32_l4_lm.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm = clCreateKernel(prog, "kernel_mul_mm_iq4_xs_f32_l4_lm", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // mul_mv_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "mul_mv_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("mul_mv_iq4_xs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_mul_mv_iq4_xs_f32 = clCreateKernel(prog, "kernel_mul_mv_iq4_xs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // mul_mv_q3_k_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4915,6 +4982,39 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemm_noshuffle_iq4_xs_q8_1_dp4a + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_xs_q8_1_dp4a.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_q8_1_dp4a.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts + " -DTILESIZE_N=8"); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + + // gemm_noshuffle_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemm_noshuffle_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemm_noshuffle_iq4_xs_f32.cl"); +#endif + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), compile_opts); + CL_CHECK((backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32 = clCreateKernel(prog, "kernel_gemm_noshuffle_iq4_xs_f32", &err), err)); + CL_CHECK(clReleaseProgram(prog)); + GGML_LOG_CONT("."); + } + // gemm_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -4930,6 +5030,41 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { GGML_LOG_CONT("."); } + // gemv_noshuffle_iq4_xs_f32 + { +#ifdef GGML_OPENCL_EMBED_KERNELS + const std::string kernel_src { + #include "gemv_noshuffle_iq4_xs_f32.cl.h" + }; +#else + const std::string kernel_src = read_file("gemv_noshuffle_iq4_xs_f32.cl"); +#endif + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_MV_NSG")) { + const int nsg = atoi(env); + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + backend_ctx->iq4_xs_mv_nsg = nsg; + } + } + for (;;) { + const std::string opts = compile_opts + " -DIQ4_XS_MV_NSG=" + std::to_string(backend_ctx->iq4_xs_mv_nsg); + cl_program prog = build_program_from_source(backend_ctx, (iq4_xs_grid_src + kernel_src).c_str(), opts); + cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_iq4_xs_f32", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (64*(size_t)backend_ctx->iq4_xs_mv_nsg <= max_workgroup) { + backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32 = kernel; + CL_CHECK(clReleaseProgram(prog)); + break; + } + CL_CHECK(clReleaseKernel(kernel)); + CL_CHECK(clReleaseProgram(prog)); + GGML_ASSERT(backend_ctx->iq4_xs_mv_nsg > 1); + backend_ctx->iq4_xs_mv_nsg /= 2; + } + GGML_LOG_CONT("."); + } + // gemv_noshuffle_q2_0_f32 { #ifdef GGML_OPENCL_EMBED_KERNELS @@ -9848,6 +9983,27 @@ struct ggml_tensor_extra_cl_iq1_m { } }; +struct ggml_tensor_extra_cl_iq4_xs { + bool noshuffle = false; + cl_mem qs = nullptr; + cl_mem sl = nullptr; + cl_mem sh = nullptr; + cl_mem d = nullptr; + + size_t size_qs = 0, size_sl = 0, size_sh = 0, size_d = 0; + + ~ggml_tensor_extra_cl_iq4_xs() { reset(); } + + void reset() { + noshuffle = false; + if (qs != nullptr) { CL_CHECK(clReleaseMemObject(qs)); qs = nullptr; } + if (sl != nullptr) { CL_CHECK(clReleaseMemObject(sl)); sl = nullptr; } + if (sh != nullptr) { CL_CHECK(clReleaseMemObject(sh)); sh = nullptr; } + if (d != nullptr) { CL_CHECK(clReleaseMemObject(d)); d = nullptr; } + size_qs = size_sl = size_sh = size_d = 0; + } +}; + struct ggml_tensor_extra_cl_q6_K { // Lower 4 bits of quantized weights. cl_mem ql = nullptr; @@ -10966,6 +11122,24 @@ static bool ggml_cl_q3_k_use_noshuffle(const ggml_backend_opencl_context * backe #endif } +static bool ggml_cl_iq4_xs_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { +#if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) + if (const char * env = getenv("GGML_OPENCL_IQ4_XS_NOSHUFFLE")) { + if (env[0] == '0') { + return false; + } + } + return tensor->type == GGML_TYPE_IQ4_XS && + backend_ctx->gpu_family == ADRENO && + ggml_is_contiguous(tensor) && !tensor->view_src && + use_adreno_kernels(backend_ctx, tensor); +#else + GGML_UNUSED(backend_ctx); + GGML_UNUSED(tensor); + return false; +#endif +} + static bool ggml_cl_iq3_s_use_noshuffle(const ggml_backend_opencl_context * backend_ctx, const ggml_tensor * tensor) { #if defined(GGML_OPENCL_SOA_Q) && defined(GGML_OPENCL_USE_ADRENO_KERNELS) if (const char * env = getenv("GGML_OPENCL_IQ3_S_NOSHUFFLE")) { @@ -11699,6 +11873,12 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te return false; } return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); + } else if (op->src[0]->type == GGML_TYPE_IQ4_XS) { + const ggml_tensor * weight = op->src[0]->view_src; + if (weight && ggml_cl_iq4_xs_use_noshuffle(backend_ctx, weight)) { + return false; + } + return op->src[1]->type == GGML_TYPE_F32 && ggml_is_contiguous_rows(op->src[1]); } else if (op->src[0]->type == GGML_TYPE_Q3_K) { const ggml_tensor * weight = op->src[0]->view_src; if (weight && ggml_cl_q3_k_use_noshuffle(backend_ctx, weight)) { @@ -12130,6 +12310,9 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs) { delete e; } + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs) { + delete e; + } for (ggml_tensor_extra_cl_q6_K * e : temp_tensor_extras_q6_K_in_use) { delete e; } @@ -12163,6 +12346,9 @@ struct ggml_backend_opencl_buffer_context { for (ggml_tensor_extra_cl_iq3_xxs * e : temp_tensor_extras_iq3_xxs_in_use) { delete e; } + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + delete e; + } for (ggml_tensor_extra_cl_q5_K * e : temp_tensor_extras_q5_K) { delete e; } @@ -12481,6 +12667,20 @@ struct ggml_backend_opencl_buffer_context { } temp_tensor_extras_iq3_xxs_in_use.push_back(extra); + extra->reset(); + return extra; + } + + ggml_tensor_extra_cl_iq4_xs * ggml_opencl_alloc_temp_tensor_extra_iq4_xs() { + ggml_tensor_extra_cl_iq4_xs * extra; + if (temp_tensor_extras_iq4_xs.empty()) { + extra = new ggml_tensor_extra_cl_iq4_xs(); + } else { + extra = temp_tensor_extras_iq4_xs.back(); + temp_tensor_extras_iq4_xs.pop_back(); + } + + temp_tensor_extras_iq4_xs_in_use.push_back(extra); extra->reset(); return extra; @@ -12611,6 +12811,10 @@ struct ggml_backend_opencl_buffer_context { temp_tensor_extras_iq3_xxs.push_back(e); } temp_tensor_extras_iq3_xxs_in_use.clear(); + for (ggml_tensor_extra_cl_iq4_xs * e : temp_tensor_extras_iq4_xs_in_use) { + temp_tensor_extras_iq4_xs.push_back(e); + } + temp_tensor_extras_iq4_xs_in_use.clear(); q8_0_soa_tensors.clear(); q4_0_soa_tensors.clear(); @@ -12663,6 +12867,8 @@ struct ggml_backend_opencl_buffer_context { std::vector temp_tensor_extras_iq3_s; std::vector temp_tensor_extras_iq3_s_in_use; std::vector temp_tensor_extras_iq3_xxs_in_use; + std::vector temp_tensor_extras_iq4_xs; + std::vector temp_tensor_extras_iq4_xs_in_use; std::vector temp_tensor_extras_q6_K; std::vector temp_tensor_extras_q6_K_in_use; @@ -14599,6 +14805,87 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer, #endif return; } + if (tensor->type == GGML_TYPE_IQ4_XS) { + GGML_ASSERT(offset == 0 && size == ggml_nbytes(tensor)); + ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; + GGML_ASSERT(extra_orig && "Tensors in OpenCL backend should have been allocated and initialized"); + + ggml_backend_opencl_buffer_context * ctx = (ggml_backend_opencl_buffer_context *) buffer->context; + ggml_tensor_extra_cl_iq4_xs * extra = ctx->ggml_opencl_alloc_temp_tensor_extra_iq4_xs(); + extra->noshuffle = ggml_cl_iq4_xs_use_noshuffle(backend_ctx, tensor); + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + const size_t n_blk = ggml_nelements(tensor) / blck; + + const size_t size_qs = n_blk * 128; + const size_t size_sl = n_blk * 4; + const size_t size_sh = n_blk * 2; + const size_t size_d = n_blk * 2; + GGML_ASSERT(size_qs + size_sl + size_sh + size_d == ggml_nbytes(tensor) && "Incorrect tensor size"); + + cl_int err; + cl_mem data_device = clCreateBuffer(context, CL_MEM_READ_WRITE, + ggml_nbytes(tensor), NULL, &err); + CL_CHECK(err); + CL_CHECK(clEnqueueWriteBuffer( + queue, data_device, CL_TRUE, 0, + ggml_nbytes(tensor), data, 0, NULL, NULL)); + + cl_buffer_region region; + region.origin = align_to(extra_orig->offset + tensor->view_offs + offset, backend_ctx->alignment); + region.size = size_qs; + CL_CHECK((extra->qs = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_qs, backend_ctx->alignment); + region.size = size_sl; + CL_CHECK((extra->sl = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sl, backend_ctx->alignment); + region.size = size_sh; + CL_CHECK((extra->sh = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + region.origin = align_to(region.origin + size_sh, backend_ctx->alignment); + region.size = size_d; + CL_CHECK((extra->d = clCreateSubBuffer(extra_orig->data_device, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err), err)); + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_convert_block_iq4_xs_noshuffle : backend_ctx->kernel_convert_block_iq4_xs; + cl_ulong nb_arg = (cl_ulong)n_blk; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra->qs)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra->sl)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra->sh)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extra->d)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t global_work_size[] = { (size_t)CEIL_DIV(n_blk, 64)*64, 1, 1 }; + size_t local_work_size[] = { 64, 1, 1 }; + + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 3, NULL, global_work_size, local_work_size, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clReleaseMemObject(data_device)); + +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + if (extra->noshuffle) { + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, extra->qs, size_qs, k/4, m, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, extra->sl, size_sl, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, extra->sh, size_sh, k/256, m, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, extra->d, size_d, k/256, m, true, true); + } +#endif + + extra->size_qs = size_qs; + extra->size_sl = size_sl; + extra->size_sh = size_sh; + extra->size_d = size_d; + + tensor->extra = extra; + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl * extra_orig = (ggml_tensor_extra_cl *)tensor->extra; GGML_ASSERT(extra_orig && "Tesnors in OpenCL backend should have been allocated and initialized"); @@ -16480,6 +16767,61 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer, CL_CHECK(clReleaseMemObject(data_device)); return; } + if (tensor->type == GGML_TYPE_IQ4_XS) { + if (tensor->view_src) { + offset += tensor->view_offs; + tensor = tensor->view_src; + } + ggml_tensor_extra_cl_iq4_xs * extra = (ggml_tensor_extra_cl_iq4_xs *)tensor->extra; + + const size_t blck = (size_t)ggml_blck_size(tensor->type); + + ggml_cl_buffer buf_unpacked; + buf_unpacked.allocate(backend_ctx->context, ggml_nbytes(tensor)); + + const size_t n_blk = (size_t)ggml_nelements(tensor) / blck; + cl_ulong nb_arg = (cl_ulong)n_blk; + + cl_mem qs = extra->qs, sl = extra->sl, sh = extra->sh, d = extra->d; +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS + ggml_cl_buffer buf_qs, buf_sl, buf_sh, buf_d; + if (extra->noshuffle) { + buf_qs.allocate(context, extra->size_qs); + buf_sl.allocate(context, extra->size_sl); + buf_sh.allocate(context, extra->size_sh); + buf_d.allocate(context, extra->size_d); + + qs = buf_qs.buffer; + sl = buf_sl.buffer; + sh = buf_sh.buffer; + d = buf_d.buffer; + const int m = tensor->ne[1]; + const int k = tensor->ne[0]; + transpose_2d_as_16b(backend_ctx, extra->qs, qs, extra->size_qs, m, k/4, true, true); + transpose_2d_as_32b(backend_ctx, extra->sl, sl, extra->size_sl, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->sh, sh, extra->size_sh, m, k/256, true, true); + transpose_2d_as_16b(backend_ctx, extra->d, d, extra->size_d, m, k/256, true, true); + } +#endif + + cl_kernel kernel = extra->noshuffle ? backend_ctx->kernel_restore_block_iq4_xs_noshuffle : backend_ctx->kernel_restore_block_iq4_xs; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &buf_unpacked.buffer)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &nb_arg)); + + size_t gws[] = { (n_blk + 63) / 64 * 64 }; + size_t lws[] = { 64 }; + cl_event evt; + CL_CHECK(clEnqueueNDRangeKernel(queue, kernel, 1, NULL, gws, lws, 0, NULL, &evt)); + CL_CHECK(clWaitForEvents(1, &evt)); + CL_CHECK(clReleaseEvent(evt)); + CL_CHECK(clEnqueueReadBuffer(queue, buf_unpacked.buffer, CL_TRUE, offset, size, data, 0, NULL, NULL)); + return; + } + if (tensor->type == GGML_TYPE_Q4_K) { ggml_tensor_extra_cl_q4_K * extra = (ggml_tensor_extra_cl_q4_K *)tensor->extra; @@ -25466,6 +25808,169 @@ static void ggml_cl_mul_mat_iq4_nl_f32_adreno(ggml_backend_t backend, const ggml #endif } +#ifdef GGML_OPENCL_USE_ADRENO_KERNELS +static bool ggml_cl_mul_mat_iq4_xs_dp4a(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + const int tile_n = n <= 8 ? 8 : 32; + cl_kernel gemm = n <= 8 ? backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a_narrow + : backend_ctx->kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a; + const char * env = getenv("GGML_OPENCL_IQ4_XS_DENSE_DP4A"); + if (!gemm || !backend_ctx->has_integer_dot || (env && env[0] == '0') || + n <= 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + return false; + } + + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem activation = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + const size_t n_blocks = (size_t)n*(k/32); + backend_ctx->prealloc_moe_qa.allocate(backend_ctx->context, (size_t)n*k); + backend_ctx->prealloc_moe_da.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + backend_ctx->prealloc_moe_sa.allocate(backend_ctx->context, n_blocks*sizeof(cl_half)); + + cl_int nb = n_blocks; + cl_kernel quant = backend_ctx->kernel_quant_a_q8_1; + CL_CHECK(clSetKernelArg(quant, 0, sizeof(cl_mem), &activation)); + CL_CHECK(clSetKernelArg(quant, 1, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(quant, 2, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(quant, 3, sizeof(cl_mem), &backend_ctx->prealloc_moe_sa.buffer)); + CL_CHECK(clSetKernelArg(quant, 4, sizeof(cl_int), &nb)); + size_t q_global[] = { CEIL_DIV(n_blocks, 64)*64 }; + size_t q_local[] = { 64 }; + backend_ctx->enqueue_ndrange_kernel(quant, 1, q_global, q_local, dst); + + CL_CHECK(clSetKernelArg(gemm, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemm, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemm, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemm, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemm, 4, sizeof(cl_mem), &backend_ctx->prealloc_moe_qa.buffer)); + CL_CHECK(clSetKernelArg(gemm, 5, sizeof(cl_mem), &backend_ctx->prealloc_moe_da.buffer)); + CL_CHECK(clSetKernelArg(gemm, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemm, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemm, 8, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemm, 9, sizeof(int), &n)); + CL_CHECK(clSetKernelArg(gemm, 10, sizeof(int), &k)); + size_t global[] = { 64, (size_t)CEIL_DIV(m, 64), (size_t)CEIL_DIV(n, tile_n) }; + size_t local[] = { 64, 1, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemm, 3, global, local, dst); + CL_CHECK(clReleaseMemObject(activation)); + return true; +} + +static void ggml_cl_mul_mat_iq4_xs_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + ggml_backend_opencl_context * backend_ctx = (ggml_backend_opencl_context *)backend->context; + ggml_tensor_extra_cl_iq4_xs * extra0 = (ggml_tensor_extra_cl_iq4_xs *)src0->extra; + ggml_tensor_extra_cl * extra1 = (ggml_tensor_extra_cl *)src1->extra; + ggml_tensor_extra_cl * extrad = (ggml_tensor_extra_cl *)dst->extra; + + cl_ulong offset1 = extra1->offset + src1->view_offs; + cl_ulong offsetd = extrad->offset + dst->view_offs; + const int m = src0->ne[1]; + const int n = src1->ne[1]; + const int k = src0->ne[0]; + + if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { + cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_iq4_xs_f32; + const int stride_d = dst->ne[0]; + const int stride_b = src1->nb[1]/sizeof(float); + CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(gemv, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(gemv, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(gemv, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(gemv, 4, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(gemv, 5, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(gemv, 6, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(gemv, 7, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(gemv, 10, sizeof(int), &stride_b)); + CL_CHECK(clSetKernelArg(gemv, 11, sizeof(int), &stride_d)); + const size_t nsg = backend_ctx->iq4_xs_mv_nsg; + size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; + size_t local[] = { 64, nsg, 1 }; + backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); + return; + } + + if (ggml_cl_mul_mat_iq4_xs_dp4a(backend, src0, src1, dst)) { + return; + } + + cl_context context = backend_ctx->context; + cl_int err; + cl_buffer_region region = { (size_t)offset1, (size_t)k*n*sizeof(float) }; + cl_mem b_sub_buf = clCreateSubBuffer(extra1->data_device, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + cl_image_format img_fmt = { CL_RGBA, CL_FLOAT }; + cl_image_desc img_desc = {}; + img_desc.image_type = CL_MEM_OBJECT_IMAGE1D_BUFFER; + img_desc.image_width = (size_t)k*n/4; + img_desc.buffer = b_sub_buf; + cl_mem b_img = clCreateImage(context, CL_MEM_READ_ONLY, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int padded_n = CEIL_DIV(n, 8)*8; + region.origin = 0; + region.size = (size_t)k*padded_n*sizeof(cl_half); + backend_ctx->prealloc_act_trans.allocate(context, region.size); + cl_mem b_sub_buf_trans = clCreateSubBuffer(backend_ctx->prealloc_act_trans.buffer, 0, CL_BUFFER_CREATE_TYPE_REGION, ®ion, &err); + CL_CHECK(err); + + img_fmt = { CL_RGBA, CL_HALF_FLOAT }; + img_desc.image_width = (size_t)k*padded_n/4; + img_desc.buffer = b_sub_buf_trans; + cl_mem b_img_trans = clCreateImage(context, 0, &img_fmt, &img_desc, nullptr, &err); + CL_CHECK(err); + + const int height_b = std::max(n/4, 1); + const int width_b = k/4; + const int padded_height_b = padded_n/4; + cl_kernel kernel = backend_ctx->kernel_transpose_32_16; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &b_img)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(int), &height_b)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(int), &width_b)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(int), &padded_height_b)); + size_t t_global[] = { (size_t)width_b, (size_t)padded_height_b }; + size_t t_local[] = { 1, 16 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 2, t_global, t_local, dst); + + kernel = backend_ctx->kernel_gemm_noshuffle_iq4_xs_f32; + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &b_img_trans)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &m)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(int), &padded_n)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &k)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &n)); + size_t global[] = { (size_t)CEIL_DIV(n, 8), (size_t)CEIL_DIV(m, 512)*128, 1 }; + size_t local[] = { 1, 128, 1 }; + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global, local, dst); + + CL_CHECK(clReleaseMemObject(b_img_trans)); + CL_CHECK(clReleaseMemObject(b_sub_buf_trans)); + CL_CHECK(clReleaseMemObject(b_img)); + CL_CHECK(clReleaseMemObject(b_sub_buf)); +} + +#endif + static void ggml_cl_mul_mat_q8_0_f32_adreno(ggml_backend_t backend, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { #ifdef GGML_OPENCL_USE_ADRENO_KERNELS GGML_ASSERT(src0); @@ -29875,6 +30380,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co ggml_tensor_extra_cl_mxfp4 * extra0_mxfp4 = (ggml_tensor_extra_cl_mxfp4 *)soa0_src->extra; ggml_tensor_extra_cl_q8_0 * extra0_q8_0 = (ggml_tensor_extra_cl_q8_0 *)soa0_src->extra; ggml_tensor_extra_cl_iq4_nl * extra0_iq4_nl = (ggml_tensor_extra_cl_iq4_nl *)soa0_src->extra; + ggml_tensor_extra_cl_iq4_xs * extra0_iq4_xs = (ggml_tensor_extra_cl_iq4_xs *)soa0_src->extra; ggml_tensor_extra_cl_q2_K * extra0_q2_K = (ggml_tensor_extra_cl_q2_K *)soa0_src->extra; ggml_tensor_extra_cl_q3_K * extra0_q3_K = (ggml_tensor_extra_cl_q3_K *)soa0_src->extra; ggml_tensor_extra_cl_iq2_s * extra0_iq2_s = (ggml_tensor_extra_cl_iq2_s *)soa0_src->extra; @@ -30278,6 +30784,12 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co return; } + // iq4_xs x fp32 + if (src0t == GGML_TYPE_IQ4_XS && src1t == GGML_TYPE_F32 && extra0_iq4_xs->noshuffle) { + ggml_cl_mul_mat_iq4_xs_f32_adreno(backend, src0, src1, dst); + return; + } + // q8_0 x fp32 if (src0t == GGML_TYPE_Q8_0 && src1t == GGML_TYPE_F32 && enable_adreno_trans_weight(backend_ctx, src0)) { @@ -30888,6 +31400,51 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); return; } + case GGML_TYPE_IQ4_XS: { + if (ne11 < 32) { + break; + } + if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { + break; + } + + kernel = backend_ctx->kernel_mul_mm_iq4_xs_f32_l4_lm; + nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; + offset0 = src0->view_offs; + + int batch_stride_a = ne00*ne01; + int batch_stride_b = ne10*ne11; + int batch_stride_d = ne0*ne1; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne02)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne10)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &batch_stride_a)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &batch_stride_b)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &batch_stride_d)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); + + size_t global_work_size[] = {(size_t)(CEIL_DIV(ne01, 64)*nth0), (size_t)CEIL_DIV(ne11, 64), (size_t)ne12*ne13}; + size_t local_work_size[] = {(size_t)nth0, 1, 1}; + + backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); + return; + } case GGML_TYPE_Q2_K: { if (ne11 < 32) { break; @@ -32264,6 +32821,70 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co CL_CHECK(clSetKernelArg(kernel, 12, sizeof(int), &ne1)); CL_CHECK(clSetKernelArg(kernel, 13, sizeof(int), &r2)); CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &r3)); +#endif // GGML_OPENCL_SOA_Q + break; + } + case GGML_TYPE_IQ4_XS: { + if (backend_ctx->gpu_family == INTEL) { + nth0 = 16; + nth1 = 1; + ndst = 4; + } else if (backend_ctx->gpu_family == ADRENO) { + nth0 = 64; + nth1 = 1; + ndst = 4; + } else { + GGML_ASSERT(false && "TODO: Unknown GPU"); + } + +#ifdef GGML_OPENCL_SOA_Q + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32_flat; + offset0 = src0->view_offs; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0_iq4_xs->qs)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_mem), &extra0_iq4_xs->sl)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra0_iq4_xs->sh)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_mem), &extra0_iq4_xs->d)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 19, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 20, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 21, sizeof(int), &r3)); +#else + kernel = backend_ctx->kernel_mul_mv_iq4_xs_f32; + + CL_CHECK(clSetKernelArg(kernel, 0, sizeof(cl_mem), &extra0->data_device)); + CL_CHECK(clSetKernelArg(kernel, 1, sizeof(cl_ulong), &offset0)); + CL_CHECK(clSetKernelArg(kernel, 2, sizeof(cl_mem), &extra1->data_device)); + CL_CHECK(clSetKernelArg(kernel, 3, sizeof(cl_ulong), &offset1)); + CL_CHECK(clSetKernelArg(kernel, 4, sizeof(cl_mem), &extrad->data_device)); + CL_CHECK(clSetKernelArg(kernel, 5, sizeof(cl_ulong), &offsetd)); + CL_CHECK(clSetKernelArg(kernel, 6, sizeof(int), &ne00)); + CL_CHECK(clSetKernelArg(kernel, 7, sizeof(int), &ne01)); + CL_CHECK(clSetKernelArg(kernel, 8, sizeof(cl_ulong), &nb01)); + CL_CHECK(clSetKernelArg(kernel, 9, sizeof(cl_ulong), &nb02)); + CL_CHECK(clSetKernelArg(kernel, 10, sizeof(cl_ulong), &nb03)); + CL_CHECK(clSetKernelArg(kernel, 11, sizeof(int), &ne12)); + CL_CHECK(clSetKernelArg(kernel, 12, sizeof(cl_ulong), &nb11)); + CL_CHECK(clSetKernelArg(kernel, 13, sizeof(cl_ulong), &nb12)); + CL_CHECK(clSetKernelArg(kernel, 14, sizeof(cl_ulong), &nb13)); + CL_CHECK(clSetKernelArg(kernel, 15, sizeof(int), &ne0)); + CL_CHECK(clSetKernelArg(kernel, 16, sizeof(int), &ne1)); + CL_CHECK(clSetKernelArg(kernel, 17, sizeof(int), &r2)); + CL_CHECK(clSetKernelArg(kernel, 18, sizeof(int), &r3)); #endif // GGML_OPENCL_SOA_Q break; } @@ -33147,7 +33768,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co src0t == GGML_TYPE_Q1_0 || src0t == GGML_TYPE_Q2_0 || src0t == GGML_TYPE_IQ4_NL || - src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS) { + src0t == GGML_TYPE_Q2_K || src0t == GGML_TYPE_Q3_K || src0t == GGML_TYPE_IQ2_S || src0t == GGML_TYPE_IQ2_XS || src0t == GGML_TYPE_IQ2_XXS || src0t == GGML_TYPE_IQ1_S || src0t == GGML_TYPE_IQ1_M || src0t == GGML_TYPE_IQ3_S || src0t == GGML_TYPE_IQ3_XXS || src0t == GGML_TYPE_IQ4_XS) { // Each SIMD group produces N_DST values in the result. Assuming each // workgroup has N_SIMDGROUP SIMD groups, then each workgroup will // produce N_DST*N_SIMDGROUP values in the result. Hence, the grid size diff --git a/ggml/src/ggml-opencl/kernels/cvt.cl b/ggml/src/ggml-opencl/kernels/cvt.cl index 143f7c24d1df..1a50ec4da678 100644 --- a/ggml/src/ggml-opencl/kernels/cvt.cl +++ b/ggml/src/ggml-opencl/kernels/cvt.cl @@ -3283,3 +3283,98 @@ kernel void kernel_restore_block_iq3_xxs( q[1] = (ushort)(sas >> 16); } } + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +kernel void kernel_convert_block_iq4_xs( + global struct block_iq4_xs * src0, + global uchar * dst_qs, + global uchar * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + for (int j = 0; j < 128; ++j) { + dst_qs[128*i + j] = src0[i].qs[j]; + } + for (int j = 0; j < 4; ++j) { + dst_sl[4*i + j] = src0[i].scales_l[j]; + } +} + +kernel void kernel_restore_block_iq4_xs( + global uchar * src_qs, + global uchar * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + for (int j = 0; j < 128; ++j) { + dst[i].qs[j] = src_qs[128*i + j]; + } + for (int j = 0; j < 4; ++j) { + dst[i].scales_l[j] = src_sl[4*i + j]; + } +} + +kernel void kernel_convert_block_iq4_xs_noshuffle( + global struct block_iq4_xs * src0, + global ushort * dst_qs, + global uint * dst_sl, + global ushort * dst_sh, + global half * dst_d, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst_d[i] = src0[i].d; + dst_sh[i] = src0[i].scales_h; + dst_sl[i] = as_uint(vload4(0, src0[i].scales_l)); + for (int sb = 0; sb < 8; ++sb) { + for (int u = 0; u < 8; ++u) { + ushort q = 0; + for (int j = 0; j < 4; ++j) { + uchar v = src0[i].qs[16*sb + 4*(u & 3) + j]; + q |= (ushort)(((v >> (4*(u >> 2))) & 15) << (4*j)); + } + dst_qs[64*i + 8*sb + u] = q; + } + } +} + +kernel void kernel_restore_block_iq4_xs_noshuffle( + global ushort * src_qs, + global uint * src_sl, + global ushort * src_sh, + global half * src_d, + global struct block_iq4_xs * dst, + ulong n_blocks +) { + const ulong i = get_global_id(0); + if (i >= n_blocks) { return; } + dst[i].d = src_d[i]; + dst[i].scales_h = src_sh[i]; + vstore4(as_uchar4(src_sl[i]), 0, dst[i].scales_l); + for (int sb = 0; sb < 8; ++sb) { + for (int j = 0; j < 16; ++j) { + const int shift = 4*(j & 3); + const ushort lo = src_qs[64*i + 8*sb + j/4]; + const ushort hi = src_qs[64*i + 8*sb + j/4 + 4]; + dst[i].qs[16*sb + j] = (uchar)(((lo >> shift) & 15) | (((hi >> shift) & 15) << 4)); + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000000..284868008d95 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,88 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_qcom_reqd_sub_group_size +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +__attribute__((qcom_reqd_sub_group_size("full"))) +#endif +kernel void kernel_gemm_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + read_only image1d_buffer_t src1, + global float * dst, + ulong offsetd, + int m, + int n, + int k, + int n_no_padding +) { + dst = (global float *)((global char *)dst + offsetd); + const int gy = get_global_id(0); + const int row = get_global_id(1) * 4; + const int n_4 = n / 4; + if (row >= m) { + return; + } + + half8 c0 = 0, c1 = 0, c2 = 0, c3 = 0; + for (int i = 0; i < k; i += 16) { + const uint4 sl = vload4(0, src0_sl + row + (i / 256) * m); + const uint4 sh = convert_uint4(vload4(0, src0_sh + row + (i / 256) * m)); + const half4 d = vload4(0, src0_d + row + (i / 256) * m); + const int sb = (i / 32) & 7; + const int4 ls = convert_int4(((sl >> (uint4)(4*sb)) & (uint4)(15)) | (((sh >> (uint4)(2*sb)) & (uint4)(3)) << (uint4)(4))) - (int4)(32); + const half4 scale = convert_half4(convert_float4(d) * convert_float4(ls)); + + for (int l = 0; l < 16; l += 4) { + const int ki = i + l; + const ushort4 q = vload4(0, src0_qs + row + (ki / 4) * m); + #pragma unroll + for (int j = 0; j < 4; ++j) { + half8 b; + b.s0123 = read_imageh(src1, gy * 2 + (ki + j) * n_4); + b.s4567 = read_imageh(src1, gy * 2 + 1 + (ki + j) * n_4); + const ushort4 indices = (q >> (ushort4)(4*j)) & (ushort4)(15); + const half4 w = (half4)(iq4_xs_values[indices.s0], iq4_xs_values[indices.s1], + iq4_xs_values[indices.s2], iq4_xs_values[indices.s3]) * scale; + c0 += b * w.s0; + c1 += b * w.s1; + c2 += b * w.s2; + c3 += b * w.s3; + } + } + } + + int idx = gy * 8 * m + row; + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s0, c1.s0, c2.s0, c3.s0), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s1, c1.s1, c2.s1, c3.s1), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s2, c1.s2, c2.s2, c3.s2), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s3, c1.s3, c2.s3, c3.s3), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s4, c1.s4, c2.s4, c3.s4), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s5, c1.s5, c2.s5, c3.s5), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s6, c1.s6, c2.s6, c3.s6), 0, dst + idx); + idx += m; + } + if (idx + 3 < m * n_no_padding) { + vstore4((float4)(c0.s7, c1.s7, c2.s7, c3.s7), 0, dst + idx); + } +} diff --git a/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl new file mode 100644 index 000000000000..f23fb1dd5785 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemm_noshuffle_iq4_xs_q8_1_dp4a.cl @@ -0,0 +1,135 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#pragma OPENCL EXTENSION cl_khr_subgroups : enable +#ifdef cl_khr_integer_dot_product +#pragma OPENCL EXTENSION cl_khr_integer_dot_product : enable +#endif + +#define QK_K 256 + +#ifndef TILESIZE_N +#define TILESIZE_N 32 +#endif + +inline uint iq4_xs_pack(ushort q) { + return iq4_xs_code(q & 15) + | (iq4_xs_code((q >> 4) & 15) << 8) + | (iq4_xs_code((q >> 8) & 15) << 16) + | (iq4_xs_code(q >> 12) << 24); +} + +// Pass staged operands by value; local dp4a operands miscompile on X2. +inline int dot8_q8a_v(uint8 qw, uint4 a0, uint4 a1) { + int r = 0; + r = dot_acc_sat_4x8packed_ss_int(qw.s0, a0.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s1, a0.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s2, a0.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s3, a0.w, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s4, a1.x, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s5, a1.y, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s6, a1.z, r); + r = dot_acc_sat_4x8packed_ss_int(qw.s7, a1.w, r); + return r; +} + +__attribute__((qcom_wave_pair_mode(1))) +kernel void kernel_gemm_noshuffle_iq4_xs_q8_1_dp4a( + __global const ushort * src0_qs, + __global const uint * src0_sl, + __global const ushort * src0_sh, + __global const half * src0_d, + __global const uint * src1_qa, + __global const half * src1_da, + __global float * dst, + ulong offsetd, + int m, + int n_no_padding, + int k +) { + dst = (global float *)((global char *)dst + offsetd); + + const uint lid = get_local_id(0); + const uint block_id_m = get_global_id(1); + const uint block_id_n = get_global_id(2); + + const uint row = block_id_m * 64 + lid; + const uint col_base = block_id_n * TILESIZE_N; + const bool row_valid = row < (uint)m; + const uint rrow = row_valid ? row : 0; + + const uint k_u = (uint)k >> 2; + const uint k_b = (uint)k >> 5; + + __local uint4 sh_qa4[TILESIZE_N][2]; + __local half sh_d[TILESIZE_N]; + +#define NGROUPS (TILESIZE_N / 4) + float4 acc[NGROUPS]; + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) acc[g] = (float4)(0.0f); + + for (uint step = 0; step < (uint)k; step += 32) { + const uint sub = step >> 5; + + const uint ib = sub >> 3; + const uint sb = sub & 7u; + const uint sl = src0_sl[rrow + ib*(uint)m]; + const uint sh = src0_sh[rrow + ib*(uint)m]; + const int ls = (int)(((sl >> (4*sb)) & 15u) | (((sh >> (2*sb)) & 3u) << 4)) - 32; + const float d_w = (float)src0_d[rrow + ib*(uint)m] * (float)ls; + const uint qsb = rrow + sub*8u*(uint)m; + + uint8 qw; + qw.s0 = iq4_xs_pack(src0_qs[qsb + 0u*(uint)m]); + qw.s1 = iq4_xs_pack(src0_qs[qsb + 1u*(uint)m]); + qw.s2 = iq4_xs_pack(src0_qs[qsb + 2u*(uint)m]); + qw.s3 = iq4_xs_pack(src0_qs[qsb + 3u*(uint)m]); + qw.s4 = iq4_xs_pack(src0_qs[qsb + 4u*(uint)m]); + qw.s5 = iq4_xs_pack(src0_qs[qsb + 5u*(uint)m]); + qw.s6 = iq4_xs_pack(src0_qs[qsb + 6u*(uint)m]); + qw.s7 = iq4_xs_pack(src0_qs[qsb + 7u*(uint)m]); + + for (uint idx = lid; idx < TILESIZE_N * 2; idx += 64) { + const uint t = idx >> 1; + const uint v = idx & 1; + const uint c = col_base + t; + sh_qa4[t][v] = (c < (uint)n_no_padding) + ? vload4(0, src1_qa + c * k_u + (step >> 2) + (v << 2)) + : (uint4)(0u); + } + if (lid < TILESIZE_N) { + const uint c = col_base + lid; + sh_d[lid] = (c < (uint)n_no_padding) ? src1_da[c * k_b + sub] : (half)0; + } + barrier(CLK_LOCAL_MEM_FENCE); + +#define LD4(arr, b) ((float4)((float)arr[(b)+0], (float)arr[(b)+1], (float)arr[(b)+2], (float)arr[(b)+3])) + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const int b = g * 4; + float4 rf; +#define TDOT4(T) dot8_q8a_v(qw, (uint4)(sh_qa4[T][0]), (uint4)(sh_qa4[T][1])) + rf.s0 = (float)TDOT4(b+0); rf.s1 = (float)TDOT4(b+1); + rf.s2 = (float)TDOT4(b+2); rf.s3 = (float)TDOT4(b+3); + acc[g] += d_w * LD4(sh_d, b) * rf; + } +#undef TDOT4 +#undef LD4 + barrier(CLK_LOCAL_MEM_FENCE); + } + + if (!row_valid) { + return; + } + + #pragma unroll + for (int g = 0; g < NGROUPS; ++g) { + const uint b = (uint)(g * 4); + const float4 a = acc[g]; + const uint c0 = col_base + b; + if (c0 + 0 < (uint)n_no_padding) dst[(c0 + 0) * (uint)m + row] = a.s0; + if (c0 + 1 < (uint)n_no_padding) dst[(c0 + 1) * (uint)m + row] = a.s1; + if (c0 + 2 < (uint)n_no_padding) dst[(c0 + 2) * (uint)m + row] = a.s2; + if (c0 + 3 < (uint)n_no_padding) dst[(c0 + 3) * (uint)m + row] = a.s3; + } +#undef NGROUPS +} diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl new file mode 100644 index 000000000000..3f71add46ca8 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_iq4_xs_f32.cl @@ -0,0 +1,85 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#define QK_K 256 + +#ifndef IQ4_XS_MV_NSG +#define IQ4_XS_MV_NSG 8 +#endif + +inline float4 iq4_xs_vals(ushort q) { + return (float4)((char)iq4_xs_code(q & 15), (char)iq4_xs_code((q >> 4) & 15), + (char)iq4_xs_code((q >> 8) & 15), (char)iq4_xs_code(q >> 12)); +} + +kernel void kernel_gemv_noshuffle_iq4_xs_f32( + global const ushort * src0_qs, + global const uint * src0_sl, + global const ushort * src0_sh, + global const half * src0_d, + global const float * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne10, + int ne0 +) { + src1 = (global const float *)((global const char *)src1 + offset1); + dst = (global float *)((global char *)dst + offsetd); + + const uint m = (uint)ne01; + const uint mh = m >> 1; + const uint lid = get_local_id(0); + const uint sgi = get_local_id(1); + const uint col = get_group_id(1); + const uint j = get_group_id(0)*64u + lid; + const uint row = j << 1; + global const float * y = src1 + (ulong)col * (uint)ne10; + + float sum0 = 0.0f, sum1 = 0.0f; + if (j < mh) { + global const uint * qs = (global const uint *)src0_qs; + global const uint * sh = (global const uint *)src0_sh; + for (uint ib = sgi; ib < (uint)ne00/QK_K; ib += IQ4_XS_MV_NSG) { + const uint base = j + ib*mh; + const uint2 sl = vload2(base, src0_sl); + const uint shp = sh[base]; + const half2 d = vload2(base, src0_d); + float acc0 = 0.0f, acc1 = 0.0f; + for (uint sb = 0; sb < 8; ++sb) { + const int ls0 = (int)(((sl.s0 >> (4*sb)) & 15u) | (((shp >> (2*sb)) & 3u) << 4)) - 32; + const int ls1 = (int)(((sl.s1 >> (4*sb)) & 15u) | (((shp >> (16 + 2*sb)) & 3u) << 4)) - 32; + float a0 = 0.0f, a1 = 0.0f; + for (uint u = 0; u < 8; ++u) { + const uint group = ib*64 + sb*8 + u; + const uint q = qs[j + group*mh]; + const float4 yv = vload4(group, y); + a0 += dot(yv, iq4_xs_vals((ushort)q)); + a1 += dot(yv, iq4_xs_vals((ushort)(q >> 16))); + } + acc0 += (float)ls0*a0; + acc1 += (float)ls1*a1; + } + sum0 += (float)d.s0*acc0; + sum1 += (float)d.s1*acc1; + } + } + +#if IQ4_XS_MV_NSG > 1 + local float2 part[IQ4_XS_MV_NSG][64]; + part[sgi][lid] = (float2)(sum0, sum1); + barrier(CLK_LOCAL_MEM_FENCE); + if (sgi != 0) { + return; + } + for (uint s = 1; s < IQ4_XS_MV_NSG; ++s) { + const float2 v = part[s][lid]; + sum0 += v.s0; + sum1 += v.s1; + } +#endif + if (j < mh) { + vstore2((float2)(sum0, sum1), 0, dst + (ulong)col*(uint)ne0 + row); + } +} diff --git a/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl new file mode 100644 index 000000000000..26fb42291ff6 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/iq4_xs_grid.cl @@ -0,0 +1,10 @@ +constant char iq4_xs_values[16] = { + -127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113, +}; + +inline uint iq4_xs_code(uint n) { + const uint lo = (n & 4u) ? 0xF6EADDCFu : 0xBFAD9881u; + const uint hi = (n & 4u) ? 0x71594535u : 0x26190D01u; + const uint t = (n & 8u) ? hi : lo; + return (t >> (8*(n & 3u))) & 255u; +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl new file mode 100644 index 000000000000..f127bd9f8a38 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mm_iq4_xs_f32_l4_lm.cl @@ -0,0 +1,186 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#define INTEL_GPU 1 +#endif + +#define QK_K 256 + +struct block_iq4_xs { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +}; + +#define LOAD_VEC_A 4 +#define LOAD_VEC_B 4 + +#define BM 64 +#define BN 64 +#define BK 32 +#ifdef INTEL_GPU +#define TM 8 +#define TN 8 +#else +#define TM 4 +#define TN 8 +#endif + +kernel void kernel_mul_mm_iq4_xs_f32_l4_lm( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global float4 * src1, + ulong offset1, + global float * dst, + ulong offsetd, + + int ne00, + int ne01, + int ne02, + int ne11, + int ne12, + + int stride_a, + int stride_b, + int stride_d, + + int batch_stride_a, + int batch_stride_b, + int batch_stride_d, + + int r2, + int r3 +) { + src1 = (global float4*)((global char*)src1 + offset1); + dst = (global float *)((global char*)dst + offsetd); + + local float buf_a[BM * BK]; + local float buf_b[BN * BK]; + + const int batch_idx = get_global_id(2); + + const int i13 = batch_idx / ne12; + const int i12 = batch_idx % ne12; + + const int i03 = i13 / r3; + const int i02 = i12 / r2; + + const int batch_idx_a = i03 * ne02 + i02; + + const int ir = get_group_id(0); + const int ic = get_group_id(1); + + const int tid = get_local_id(0); + const int th_r = tid % (BM / TM); + const int th_c = tid / (BM / TM); + + const int loadr_a = get_local_id(0) % (BK / LOAD_VEC_A); + const int loadc_a = get_local_id(0) / (BK / LOAD_VEC_A); + const int loadr_b = get_local_id(0) % (BK / LOAD_VEC_B); + const int loadc_b = get_local_id(0) / (BK / LOAD_VEC_B); + + const int loadstride_a = get_local_size(0) * LOAD_VEC_A / BK; + const int loadstride_b = get_local_size(0) * LOAD_VEC_B / BK; + + // pos_a counts elements, not blocks - one iq4_xs super block holds QK_K of them + int pos_a = batch_idx_a * batch_stride_a + ir * BM * stride_a; + int pos_b = (batch_idx * batch_stride_b + ic * BN * stride_b) / LOAD_VEC_B; + + float sums[TM * TN]; + float cache_a[TM]; + float cache_b[TN]; + + for (int i = 0; i < TM * TN; i++) { + sums[i] = 0.0f; + } + + for (int block = 0; block < ne00; block += BK) { + for (int l = 0; l < BM; l += loadstride_a) { + if (ir*BM + loadc_a + l < ne01) { + int idx = pos_a + (loadc_a + l) * stride_a + loadr_a * LOAD_VEC_A; + int ib = idx / QK_K; + ib += offset0/sizeof(struct block_iq4_xs); + int e = idx % QK_K; + + int ib32 = e >> 5; + int j = e & 15; + int shift = (e & 16) >> 2; + int ls = ((src0_sl[4*ib + ib32/2] >> (4*(ib32 & 1))) & 15) | (((src0_sh[ib] >> (2*ib32)) & 3) << 4); + float db = (float)src0_d[ib] * (float)(ls - 32); + uchar4 q = vload4(0, src0_qs + 128*ib + 16*ib32 + j); + q = (q >> (uchar4)(shift)) & (uchar4)(15); + + float4 v1; + v1.s0 = db * (float)iq4_xs_values[q.s0]; + v1.s1 = db * (float)iq4_xs_values[q.s1]; + v1.s2 = db * (float)iq4_xs_values[q.s2]; + v1.s3 = db * (float)iq4_xs_values[q.s3]; + + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = v1.s0; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = v1.s1; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = v1.s2; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = v1.s3; + } else { + buf_a[(loadr_a * LOAD_VEC_A + 0) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 1) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 2) * BM + loadc_a + l] = 0.0f; + buf_a[(loadr_a * LOAD_VEC_A + 3) * BM + loadc_a + l] = 0.0f; + } + } + + for (int l = 0; l < BN; l += loadstride_b) { + if (ic*BN + loadc_b + l < ne11) { + int idx = pos_b + (loadc_b + l) * stride_b / LOAD_VEC_B + loadr_b; + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = src1[idx].s0; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = src1[idx].s1; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = src1[idx].s2; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = src1[idx].s3; + } else { + buf_b[(loadr_b * LOAD_VEC_B + 0) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 1) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 2) * BN + loadc_b + l] = 0.0f; + buf_b[(loadr_b * LOAD_VEC_B + 3) * BN + loadc_b + l] = 0.0f; + } + } + + barrier(CLK_LOCAL_MEM_FENCE); + + pos_a += BK; + pos_b += BK / LOAD_VEC_B; + + for (int i = 0; i < BK; i++) { + for (int j = 0; j < TM; j++) { + cache_a[j] = buf_a[(i) * BM + th_r * TM + j]; + } + + for (int j = 0; j < TN; j++) { + cache_b[j] = buf_b[(i) * BN + th_c * TN + j]; + } + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + const int sums_idx = cc*TM + cr; + sums[sums_idx] = mad(cache_a[cr], cache_b[cc], sums[sums_idx]); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + } + + const int dr = ir * BM + th_r * TM; + const int dc = ic * BN + th_c * TN; + + const ulong offsets = batch_idx * batch_stride_d; + + for (int cc = 0; cc < TN; cc++) { + for (int cr = 0; cr < TM; cr++) { + if (dr + cr < ne01 && dc + cc < ne11) { + dst[offsets + (dc + cc) * stride_d + dr + cr] = sums[cc * TM + cr]; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl new file mode 100644 index 000000000000..2fe0044ebdc1 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32( + global char * src0, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 = src0 + offset0; + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03; + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global block_iq4_xs * x = (global block_iq4_xs *) (src0 + offset_src0); + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + global char * xrow = (global char *)(x + ib); + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + global block_iq4_xs * xb = (global block_iq4_xs *)(xrow + row*nb01); + + int ls = ((xb->scales_l[it/2] >> (4*(it%2))) & 15) | (((xb->scales_h >> (2*it)) & 3) << 4); + float db = (float)xb->d * (float)(ls - 32); + global uchar * qs = xb->qs + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl new file mode 100644 index 000000000000..4426137f0b48 --- /dev/null +++ b/ggml/src/ggml-opencl/kernels/mul_mv_iq4_xs_f32_flat.cl @@ -0,0 +1,134 @@ +#pragma OPENCL EXTENSION cl_khr_fp16 : enable + +#ifdef cl_intel_required_subgroup_size +#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable +#define INTEL_GPU 1 +#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16))) +#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32))) +#elif defined(cl_qcom_reqd_sub_group_size) +#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable +#define ADRENO_GPU 1 +#define REQD_SUBGROUP_SIZE_64 __attribute__((qcom_reqd_sub_group_size("half"))) +#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full"))) +#endif + +#define QK_K 256 + +typedef struct { + half d; + ushort scales_h; + uchar scales_l[QK_K/64]; + uchar qs[QK_K/2]; +} block_iq4_xs; + +#undef N_DST +#undef N_SIMDGROUP +#undef N_SIMDWIDTH + +#ifdef INTEL_GPU +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 16 +#elif defined (ADRENO_GPU) +#define N_DST 4 +#define N_SIMDGROUP 1 +#define N_SIMDWIDTH 64 +#endif + +#undef BLOCK_STRIDE +// 8 threads cover one super block, one 32 element sub block each +#define BLOCK_STRIDE (N_SIMDWIDTH/8) + +#ifdef INTEL_GPU +REQD_SUBGROUP_SIZE_16 +#elif defined (ADRENO_GPU) +REQD_SUBGROUP_SIZE_64 +#endif +kernel void kernel_mul_mv_iq4_xs_f32_flat( + global uchar * src0_qs, + global uchar * src0_sl, + global ushort * src0_sh, + global half * src0_d, + ulong offset0, + global char * src1, + ulong offset1, + global char * dst, + ulong offsetd, + int ne00, + int ne01, + ulong nb01, + ulong nb02, + ulong nb03, + int ne12, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src1 = src1 + offset1; + dst = dst + offsetd; + + int ix = get_sub_group_local_id()/8; // super block index + int it = get_sub_group_local_id()%8; // sub block inside the super block + + int nb = ne00/QK_K; + + int r0 = get_group_id(0); + int r1 = get_group_id(1); + int im = get_group_id(2); + int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST; + + int i12 = im%ne12; + int i13 = im/ne12; + + ulong offset_src0 = (offset0 + first_row*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03)/sizeof(block_iq4_xs); + ulong row_stride = nb01/sizeof(block_iq4_xs); + ulong offset_src1 = r1*nb11 + (i12 )*nb12 + (i13 )*nb13; + + global float * y = (global float *) (src1 + offset_src1); + + float yl[32]; + float sumf[N_DST] = {0.f}; + float all_sum; + + global float * y4 = y + ix * QK_K + 32 * it; + + for (int ib = ix; ib < nb; ib += BLOCK_STRIDE) { + for (int i = 0; i < 32; ++i) { + yl[i] = y4[i]; + } + + // only ne01 output rows exist; reading past them can pull in an inf scale + for (int row = 0; row < N_DST && first_row + row < ne01; row++) { + const ulong b = offset_src0 + ib + row*row_stride; + + int ls = ((src0_sl[4*b + it/2] >> (4*(it%2))) & 15) | (((src0_sh[b] >> (2*it)) & 3) << 4); + float db = (float)src0_d[b] * (float)(ls - 32); + global uchar * qs = src0_qs + 128*b + 16*it; + + float acc = 0.f; + for (int j = 0; j < 16; ++j) { + acc += yl[j] * (float)iq4_xs_values[qs[j] & 15]; + acc += yl[j+16] * (float)iq4_xs_values[qs[j] >> 4]; + } + + sumf[row] += db * acc; + } + + y4 += BLOCK_STRIDE * QK_K; + } + + global float * dst_f32 = (global float *) dst + im*ne0*ne1 + r1*ne0; + + for (int row = 0; row < N_DST; ++row) { + all_sum = sub_group_reduce_add(sumf[row]); + if (first_row + row < ne01) { + if (get_sub_group_local_id() == 0) { + dst_f32[first_row + row] = all_sum; + } + } + } +} From 00f113eceb2758158582d56f98bf476b2fe3ed32 Mon Sep 17 00:00:00 2001 From: hesiru Date: Tue, 3 Nov 2026 14:06:03 +0800 Subject: [PATCH 008/286] cuda : prevent premature eviction of graphs about to be used (#30662) * cuda : prevent premature eviction of graphs about to be used * add last_graph_use_time for entire cuda graph map and testcase for cuda graph Assisted-by: Codex * delete the test --- ggml/src/ggml-cuda/common.cuh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index a46c4d3f096f..00476f0fd134 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1469,16 +1469,21 @@ struct ggml_backend_cuda_context { // when the computation is split across CPU/GPU (e.g., with --n-cpu-moe) std::unordered_map> cuda_graphs; - int64_t last_graph_eviction_sweep = 0; + int64_t next_graph_eviction_time = 0; + int64_t last_graph_use_time = 0; - ggml_cuda_graph * cuda_graph(const void * first_node_ptr) { - const int64_t time_now = ggml_time_us(); + ggml_cuda_graph * cuda_graph(const void * first_node_ptr, int64_t time_now = ggml_time_us()) { + // Give all cached graphs 10s to be reused after an idle period, without updating each graph. + if (!cuda_graphs.empty() && time_now - last_graph_use_time >= 10'000'000) { + next_graph_eviction_time = time_now + 10'000'000; + } + last_graph_use_time = time_now; - // sweep every 5s, evicting cuda graphs unused for >=10s - if (time_now - last_graph_eviction_sweep >= 5'000'000) { - last_graph_eviction_sweep = time_now; + // Sweep every 5s during active use, evicting graphs unused for at least 10s. + if (time_now >= next_graph_eviction_time) { + next_graph_eviction_time = time_now + 5'000'000; for (auto it = cuda_graphs.begin(); it != cuda_graphs.end(); ) { - if (time_now - it->second->last_used_time >= 10'000'000) { + if (it->first != first_node_ptr && time_now - it->second->last_used_time >= 10'000'000) { it = cuda_graphs.erase(it); } else { ++it; From 0f663d2471bb470c1979999daf7a05ec8fba5673 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 14:08:42 +0800 Subject: [PATCH 009/286] dsv4.1: fix llama-context tests with partial offload (#31292) --- src/llama-kv-cache-dsv4.cpp | 48 ++++++++++++++++++++++--------------- src/llama-kv-cache-dsv4.h | 7 +++++- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index d5506f7a7bf2..ef1be94a27a9 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -428,25 +428,6 @@ static void dsv4_state_read_k_cache( dsv4_state_read_tensor_streams(io, kv->get_k_storage(il), kv_size, n_rows_ref, s0, ns); } - - if (n_rows_ref == 0) { - return; - } - - // refill the mirrors from their sources; a mirror gets the same rows as its source, so a whole copy is exact - for (uint32_t il : kv->get_layer_ids()) { - const int32_t src = kv_src[il]; - if (src == (int32_t) il) { - continue; - } - ggml_tensor * t_src = kv->get_k_storage(src); - ggml_tensor * t_dst = kv->get_k_storage(il); - if (t_src->type != t_dst->type || !ggml_are_same_shape(t_src, t_dst)) { - throw std::runtime_error("DSV4 K-cache mirror layout mismatch"); - } - - ggml_backend_tensor_copy(t_src, t_dst); - } } static std::string dsv4_plan_positions(const std::vector & values) { @@ -1797,6 +1778,9 @@ void llama_kv_cache_dsv4::state_read(llama_io_read_i & io, llama_seq_id seq_id, dsv4_state_read_k_cache(io, kv_csa.get(), seq_id, flags, comp_il_src); dsv4_state_read_k_cache(io, kv_hca.get(), seq_id, flags, comp_il_src); dsv4_state_read_k_cache(io, kv_lid.get(), seq_id, flags, comp_il_src); + + // the io can defer the tensor reads until it is destroyed, so the mirrors are refilled before the next graph + comp_mirrors_stale = true; } csa_state->state_read(io, seq_id, flags); @@ -1882,6 +1866,30 @@ void llama_kv_cache_dsv4::update_rs_for_ubatches(const std::vector } } +void llama_kv_cache_dsv4::refill_comp_mirrors() { + if (!comp_mirrors_stale) { + return; + } + + comp_mirrors_stale = false; + + // a mirror gets the same rows as its source, so a whole copy is exact + for (const llama_kv_cache * kv : { kv_csa.get(), kv_hca.get(), kv_lid.get() }) { + for (uint32_t il : kv->get_layer_ids()) { + const int32_t src = comp_il_src[il]; + if (src == (int32_t) il) { + continue; + } + + ggml_tensor * t_src = kv->get_k_storage(src); + ggml_tensor * t_dst = kv->get_k_storage(il); + GGML_ASSERT(t_src->type == t_dst->type && ggml_are_same_shape(t_src, t_dst)); + + ggml_backend_tensor_copy(t_src, t_dst); + } + } +} + void llama_kv_cache_dsv4::clear_compressed(llama_seq_id seq_id, bool data) { if (seq_id < 0) { kv_csa->clear(data); @@ -2281,6 +2289,8 @@ bool llama_kv_cache_dsv4_context::apply() { lid_state->apply_copies(sc_info_lid); } + kv->refill_comp_mirrors(); + return res; } diff --git a/src/llama-kv-cache-dsv4.h b/src/llama-kv-cache-dsv4.h index 1c8d23ed2801..50cb11b626ad 100644 --- a/src/llama-kv-cache-dsv4.h +++ b/src/llama-kv-cache-dsv4.h @@ -157,6 +157,9 @@ class llama_kv_cache_dsv4 : public llama_memory_i { const std::vector & get_rs_idx() const; void update_rs_for_ubatches(const std::vector & ubatches); + // copy the sources into their mirrors after a state read + void refill_comp_mirrors(); + private: llama_hparams hparams_raw; llama_hparams hparams_csa; @@ -178,6 +181,8 @@ class llama_kv_cache_dsv4 : public llama_memory_i { // [n_layer] layer -> layer whose compressed-KV storage it reads (itself, its V4.1 kv source or a mirror), -1 = none std::vector comp_read; + bool comp_mirrors_stale = false; + std::unique_ptr kv_raw; std::unique_ptr kv_csa; std::unique_ptr kv_hca; @@ -422,5 +427,5 @@ class llama_kv_cache_dsv4_context : public llama_memory_context_i { const llama_memory_status status; - const llama_kv_cache_dsv4 * kv = nullptr; + llama_kv_cache_dsv4 * kv = nullptr; }; From ca8daad199a9c12c5b54643b974c640939a0f7bd Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 3 Nov 2026 08:11:06 +0200 Subject: [PATCH 010/286] llama : drop the redundant cnv from llm_attn_mask (#31558) The dspark block mask is now derived at input-fill time: llm_attn_mask::block_uniform fills every token of a sequence with the mask row of its first token, so the graph-level reshape/repeat in dflash is gone and cnv always aliases raw. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) --- src/llama-graph.cpp | 37 +++++++++-------------------------- src/llama-graph.h | 5 +++-- src/llama-kv-cache.cpp | 18 ++++++++++++++--- src/models/clef.cpp | 2 -- src/models/deepseek32.cpp | 4 ++-- src/models/deepseek4.cpp | 6 +++--- src/models/deepseek41.cpp | 2 +- src/models/dflash.cpp | 22 ++++----------------- src/models/dots3note.cpp | 4 ++-- src/models/glm-dsa.cpp | 4 ++-- src/models/hy-v4.cpp | 4 ++-- src/models/lfm2.cpp | 1 - src/models/minimax-m3.cpp | 2 +- src/models/mistral-large4.cpp | 4 ++-- 14 files changed, 46 insertions(+), 69 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe8bdebe5e2c..f0463f849f1b 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -124,8 +124,6 @@ static llm_attn_mask build_attn_inp_attn_mask( ggml_set_name(attn_mask.raw, "attn_inp_kq_mask"); } - attn_mask.cnv = attn_mask.raw; - return attn_mask; } @@ -3131,16 +3129,11 @@ llm_graph_input_attn_no_cache * llm_graph_context::build_attn_inp_no_cache() con ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) { inp->self_attn_mask_swa.raw = ggml_new_tensor_4d(ctx0, type_mask, n_kv, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask_swa.raw); - - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } else { inp->self_attn_mask_swa.raw = nullptr; - inp->self_attn_mask_swa.cnv = nullptr; } return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); @@ -3169,7 +3162,7 @@ ggml_tensor * llm_graph_context::build_attn( const bool is_swa = hparams.is_swa(il); - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // [TAG_NO_CACHE_PAD] @@ -3296,7 +3289,7 @@ ggml_tensor * llm_graph_context::build_attn( v = mctx_cur->get_v(ctx0, il); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, kv_rows, sinks, v_mla, 0, kq_scale, il); @@ -3341,7 +3334,6 @@ static std::unique_ptr build_attn_inp_k_impl( inp->self_k_idxs = mctx_cur->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } return inp; @@ -3384,7 +3376,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = inp->self_attn_mask.cnv; + const auto & kq_mask = inp->self_attn_mask.raw; const auto & kv_rows = inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3465,9 +3457,9 @@ ggml_tensor * llm_graph_context::build_attn( // top_k indexes the cells, and the indexer mask of the same type is built the same way // let's use it, so when no other layer reads the MLA mask is not allocated - ggml_tensor * kq_mask = inp->self_attn_mask_mla.cnv; - if (inp->self_attn_mask_lid.cnv->type == kq_mask->type) { - kq_mask = inp->self_attn_mask_lid.cnv; + ggml_tensor * kq_mask = inp->self_attn_mask_mla.raw; + if (inp->self_attn_mask_lid.raw->type == kq_mask->type) { + kq_mask = inp->self_attn_mask_lid.raw; } ggml_tensor * kq_mask_top_k = nullptr; @@ -3589,7 +3581,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_v(ctx0, v_cur, v_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3660,7 +3652,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, k_cur, k_idxs, il)); } - const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.cnv : inp->self_attn_mask.cnv; + const auto & kq_mask = is_swa ? inp->self_attn_mask_swa.raw : inp->self_attn_mask.raw; const auto & kv_rows = is_swa ? inp->self_attn_mask_swa.rows : inp->self_attn_mask.rows; // MLA-style attention: the cached K is used as V @@ -3697,8 +3689,6 @@ llm_graph_input_attn_cross * llm_graph_context::build_attn_inp_cross() const { inp->cross_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_enc, n_tokens, 1, 1); ggml_set_input(inp->cross_attn_mask.raw); - inp->cross_attn_mask.cnv = inp->cross_attn_mask.raw; - return (llm_graph_input_attn_cross *) res->add_input(std::move(inp)); } @@ -3721,7 +3711,7 @@ ggml_tensor * llm_graph_context::build_attn( ggml_build_forward_expand(gf, k_cur); ggml_build_forward_expand(gf, v_cur); - const auto & kq_mask = inp->cross_attn_mask.cnv; + const auto & kq_mask = inp->cross_attn_mask.raw; const auto & kv_rows = inp->cross_attn_mask.rows; ggml_tensor * q = q_cur; @@ -3759,7 +3749,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( inp->self_k_idxs_mla = mctx_cur->get_mla()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_mla.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_mla(), ubatch, cparams); - inp->self_attn_mask_mla.cnv = inp->self_attn_mask_mla.raw; } { @@ -3770,7 +3759,6 @@ static std::unique_ptr build_attn_inp_k_dsa_impl( cparams_copy.flash_attn = cparams.fused_lid; inp->self_attn_mask_lid.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_lid(), ubatch, cparams_copy); - inp->self_attn_mask_lid.cnv = inp->self_attn_mask_lid.raw; inp->self_k_rot_lid = mctx_cur->get_lid()->get_k_rot(); } @@ -3797,7 +3785,6 @@ llm_graph_input_attn_k_dsa_iswa * llm_graph_context::build_attn_inp_k_dsa_iswa() inp_swa->self_k_idxs = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp_swa->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp_swa->self_attn_mask.cnv = inp_swa->self_attn_mask.raw; auto inp = std::make_unique(std::move(inp_dsa), std::move(inp_swa), mctx_cur); @@ -3819,7 +3806,6 @@ llm_graph_input_attn_kv_msa * llm_graph_context::build_attn_inp_kv_msa(bool msa_ inp->self_v_idxs = mctx_base->build_input_v_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_base, ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } inp->self_k_rot = mctx_base->get_k_rot(); @@ -3874,7 +3860,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs = mctx_cur->get_base()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_base(), ubatch, cparams); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; } { @@ -3883,7 +3868,6 @@ llm_graph_input_attn_k_iswa * llm_graph_context::build_attn_inp_k_iswa() const { inp->self_k_idxs_swa = mctx_cur->get_swa()->build_input_k_idxs(ctx0, ubatch); inp->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, mctx_cur->get_swa(), ubatch, cparams); - inp->self_attn_mask_swa.cnv = inp->self_attn_mask_swa.raw; } inp->self_k_rot = mctx_cur->get_base()->get_k_rot(); @@ -3905,7 +3889,6 @@ llm_graph_input_dsv4 * llm_graph_context::build_inp_dsv4() const { inp_raw->self_k_idxs = raw_ctx->build_input_k_idxs(ctx0, ubatch); inp_raw->self_attn_mask.raw = dsv4_build_raw_kq_mask(ctx0, raw_ctx, ubatch, cparams, n_stream); - inp_raw->self_attn_mask.cnv = inp_raw->self_attn_mask.raw; inp_raw->self_k_rot = raw_ctx->get_k_rot(); auto inp = std::make_unique(cparams, std::move(inp_raw), mctx_cur); @@ -4089,7 +4072,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs = attn_ctx->get_base()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_base(), ubatch, cparams); - inp_attn->self_attn_mask.cnv = inp_attn->self_attn_mask.raw; } { @@ -4097,7 +4079,6 @@ llm_graph_input_mem_hybrid_iswa * llm_graph_context::build_inp_mem_hybrid_iswa() inp_attn->self_v_idxs_swa = attn_ctx->get_swa()->build_input_v_idxs(ctx0, ubatch); inp_attn->self_attn_mask_swa.raw = build_attn_inp_kq_mask(ctx0, attn_ctx->get_swa(), ubatch, cparams); - inp_attn->self_attn_mask_swa.cnv = inp_attn->self_attn_mask_swa.raw; } inp_attn->self_k_rot = attn_ctx->get_base()->get_k_rot(); diff --git a/src/llama-graph.h b/src/llama-graph.h index 37b47acbb30a..ea2cf9f873cf 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -333,12 +333,13 @@ class llm_graph_input_cross_embd : public llm_graph_input_i { // the mask has two shapes depending on whether `rows` is used: // rows != nullptr: [n_kv_rows, n_batch/n_slices, 1, n_slices] // rows == nullptr: [n_kv, n_batch/n_stream, 1, n_stream] -// TODO: cnv is redundant (always aliases raw) except dflash overwrites it with a derived block mask [TAG_ATTN_MASK_CNV] struct llm_attn_mask { ggml_tensor * raw = nullptr; - ggml_tensor * cnv = nullptr; // same shape ggml_tensor * rows = nullptr; // I32 [n_kv_rows, n_slices] uint32_t n_kv = 0; + + // fill every token of a sequence with the mask row of its first token (dspark draft blocks) + bool block_uniform = false; }; class llm_graph_input_attn_no_cache : public llm_graph_input_i { diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index d9f46f0adccf..06e218ba65af 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1683,6 +1683,9 @@ struct args_set_input_attn_mask { // cell of each mask column per stream, nullptr if the columns are the cells const int32_t * rows; + + // every token of a sequence reuses the mask row of the first one (dspark draft blocks) + bool block_uniform; }; template @@ -1709,10 +1712,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * llama_pos seq_pos_min[LLAMA_MAX_SEQ]; std::fill(seq_pos_min, seq_pos_min + LLAMA_MAX_SEQ, INT32_MAX); + // first token of each sequence, used when the mask rows are shared within a sequence + uint32_t seq_first[LLAMA_MAX_SEQ]; + std::fill(seq_first, seq_first + LLAMA_MAX_SEQ, ubatch->n_tokens); + for (uint32_t i = 0; i < ubatch->n_tokens; ++i) { const llama_seq_id seq_id = ubatch->seq_id[i][0]; seq_pos_min[seq_id] = std::min(seq_pos_min[seq_id], ubatch->pos[i]); + seq_first[seq_id] = std::min(seq_first[seq_id], i); } for (uint32_t s = 0; s < n_stream; ++s) { @@ -1729,12 +1737,15 @@ static void set_input_attn_mask_impl(const args_set_input_attn_mask & args, T * const auto & cells = v_cells.at(seq_to_stream[seq_id]); + // shared mask rows: every query of the sequence uses the row of its first token + const uint32_t iq = args.block_uniform ? seq_first[seq_id] : i; + llama_pos p0 = -1; - const llama_pos p1 = ubatch->pos[i]; + const llama_pos p1 = ubatch->pos[iq]; // for M-RoPE - const llama_pos p1_x = is_2d ? ubatch->pos[i + ubatch->n_tokens*2] : 0; - const llama_pos p1_y = is_2d ? ubatch->pos[i + ubatch->n_tokens] : 0; + const llama_pos p1_x = is_2d ? ubatch->pos[iq + ubatch->n_tokens*2] : 0; + const llama_pos p1_y = is_2d ? ubatch->pos[iq + ubatch->n_tokens] : 0; const uint64_t idst = n_kv*i; @@ -1941,6 +1952,7 @@ void llama_kv_cache::set_input_attn_mask(const llm_attn_mask & attn_mask, const /*.n_stream =*/ n_stream, /*.n_tps =*/ n_tps, /*.rows =*/ rows, + /*.block_uniform =*/ attn_mask.block_uniform, }; if (dst->type == GGML_TYPE_F16) { diff --git a/src/models/clef.cpp b/src/models/clef.cpp index e9be0ec141af..89d9419b83d8 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -352,8 +352,6 @@ llm_graph_input_attn_no_cache * llama_model_clef::graph::build_attn_inp_causal() ggml_set_input(inp->self_attn_mask.raw); cb(inp->self_attn_mask.raw, "self_kq_mask", -1); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; - return (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } diff --git a/src/models/deepseek32.cpp b/src/models/deepseek32.cpp index 22f516a481ea..450022016b77 100644 --- a/src/models/deepseek32.cpp +++ b/src/models/deepseek32.cpp @@ -271,7 +271,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -305,7 +305,7 @@ llama_model_deepseek32::graph::graph(const llama_model & model, const llm_graph_ cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/deepseek4.cpp b/src/models/deepseek4.cpp index 85fe141fd87a..1207c3bb73b3 100644 --- a/src/models/deepseek4.cpp +++ b/src/models/deepseek4.cpp @@ -759,7 +759,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_csa_lid_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, csa_k, 2); cb(k_all, "csa_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * csa_mask = build_top_k_mask(inp_csa.kq_mask, top_k, "csa_top_k_mask", il); ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, csa_mask, 0); @@ -815,7 +815,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_hca_attention( ggml_tensor * k_all = ggml_concat(ctx0, raw_k, hca_k, 2); cb(k_all, "hca_k_all", il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; ggml_tensor * hca_mask = inp_hca.kq_mask; ggml_tensor * kq_mask = ggml_concat(ctx0, raw_mask, hca_mask, 0); @@ -853,7 +853,7 @@ ggml_tensor * llama_model_deepseek4::graph::build_raw_attention( ggml_build_forward_expand(gf, mctx_cur->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); - ggml_tensor * kq_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * kq_mask = inp_attn->self_attn_mask.raw; ggml_tensor * k = mctx_cur->get_k(ctx0, il); diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index d8299897043b..c4ca567e71e5 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -642,7 +642,7 @@ ggml_tensor * llama_model_deepseek41::graph::build_attention41( ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, kv, inp_attn->get_k_idxs(), il)); ggml_tensor * raw_k = inp_attn->mctx->get_k(ctx0, il); - ggml_tensor * raw_mask = inp_attn->self_attn_mask.cnv; + ggml_tensor * raw_mask = inp_attn->self_attn_mask.raw; const int32_t topk_src = hparams.dsv41_topk_src[il]; const int64_t k_sel = top_k->ne[0]; diff --git a/src/models/dflash.cpp b/src/models/dflash.cpp index fdd67e01785b..0de2f38ecf93 100644 --- a/src/models/dflash.cpp +++ b/src/models/dflash.cpp @@ -1391,22 +1391,6 @@ llama_model_dflash::graph_dsv4::graph_dsv4(const llama_model & model, const llm_ // * embd batch -> writes every stage's kv_norm(wkv(main_proj(target features))) into its window ring // * token batch -> [anchor, noise...] block through the stages, single-pass mHC, no per-head q norm // window of a block = window of its first query, see n_swa -// TODO: move this to a new type of `llm_graph_input` and implement this in `set_input` [TAG_ATTN_MASK_CNV] -static void build_dspark_block_mask(llm_graph_context & g, llm_graph_input_attn_k_iswa * inp_attn) { - ggml_context * ctx0 = g.ctx0; - - ggml_tensor * m = inp_attn->self_attn_mask_swa.cnv; - const int64_t n_tokens = g.n_tokens; - const int64_t n_blocks = g.ubatch.n_seqs_unq; - const int64_t bd = n_blocks > 0 ? n_tokens / n_blocks : 0; - if (m && bd > 1 && n_tokens % n_blocks == 0 && m->ne[2] == 1 && m->ne[1] % bd == 0 && ggml_is_contiguous(m)) { - ggml_tensor * m4 = ggml_reshape_4d(ctx0, m, m->ne[0], bd, m->ne[1]/bd, m->ne[3]); - ggml_tensor * r0 = ggml_view_4d(ctx0, m4, m4->ne[0], 1, m4->ne[2], m4->ne[3], m4->nb[1], m4->nb[2], m4->nb[3], 0); - m4 = ggml_repeat(ctx0, r0, m4); - inp_attn->self_attn_mask_swa.cnv = ggml_reshape_4d(ctx0, m4, m->ne[0], m->ne[1], 1, m->ne[3]); - g.cb(inp_attn->self_attn_mask_swa.cnv, "dspark_block_mask", -1); - } -} llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const llm_graph_params & params) : llama_model_deepseek41::graph(params) { @@ -1496,7 +1480,8 @@ llama_model_dflash::graph_dsv41::graph_dsv41(const llama_model & model, const ll inpL = ggml_repeat_4d(ctx0, inpL, n_embd, hc, n_tokens, 1); cb(inpL, "hc_init", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; ggml_tensor * pre_prev = nullptr; @@ -1743,7 +1728,8 @@ llama_model_dflash::graph_mistral_large4::graph_mistral_large4(const llama_model } cb(inpL, "inp_noise_embd", -1); - build_dspark_block_mask(*this, inp_attn); + // the whole block shares the mask row of its first query (the anchor) + inp_attn->self_attn_mask_swa.block_uniform = true; for (int il = 0; il < n_layer; ++il) { const auto & layer = model.layers[il]; diff --git a/src/models/dots3note.cpp b/src/models/dots3note.cpp index df6fee7e4b50..c848b0fe2d4e 100644 --- a/src/models/dots3note.cpp +++ b/src/models/dots3note.cpp @@ -256,7 +256,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn->get_dsa()->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -286,7 +286,7 @@ llama_model_dots3note::graph::graph(const llama_model & model, const llm_graph_p indexer_score = ggml_cont(ctx0, ggml_permute(ctx0, indexer_score, 2, 1, 0, 3)); cb(indexer_score, "indexer_score", il); - ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn->get_dsa()->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/glm-dsa.cpp b/src/models/glm-dsa.cpp index 0bf7a7c853a3..146112d365b9 100644 --- a/src/models/glm-dsa.cpp +++ b/src/models/glm-dsa.cpp @@ -305,7 +305,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -339,7 +339,7 @@ llama_model_glm_dsa::graph::graph(const llama_model & model, const llm_graph_par cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } diff --git a/src/models/hy-v4.cpp b/src/models/hy-v4.cpp index 06833e20bb42..945e0cc37ce8 100644 --- a/src/models/hy-v4.cpp +++ b/src/models/hy-v4.cpp @@ -395,7 +395,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( ggml_tensor * score = nullptr; if (cparams.fused_lid) { - score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_lightning_indexer(ctx0, iq, ik, iw, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, score, il}); } else { @@ -408,7 +408,7 @@ ggml_tensor * llama_model_hy_v4::graph::build_indexer_top_k( score = ggml_mul(ctx0, score, iw); score = ggml_sum_rows(ctx0, score); score = ggml_cont(ctx0, ggml_permute(ctx0, score, 2, 1, 0, 3)); - score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.cnv); + score = ggml_add(ctx0, score, inp_attn_dsa->self_attn_mask_lid.raw); cb(score, "indexer_score", il); } diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index 3488b6c775a8..e18937faf3ca 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -332,7 +332,6 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons auto inp = std::make_unique(hparams, cparams, is_head); inp->self_attn_mask.raw = ggml_new_tensor_4d(ctx0, type_mask, n_tokens, n_tokens, 1, 1); ggml_set_input(inp->self_attn_mask.raw); - inp->self_attn_mask.cnv = inp->self_attn_mask.raw; inp_attn[is_head] = (llm_graph_input_attn_no_cache *) res->add_input(std::move(inp)); } diff --git a/src/models/minimax-m3.cpp b/src/models/minimax-m3.cpp index 0da9d5a44aa8..6d1528096166 100644 --- a/src/models/minimax-m3.cpp +++ b/src/models/minimax-m3.cpp @@ -262,7 +262,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_ if (msa_enabled) { const auto * mctx_msa = static_cast(mctx); - msa_kqm = inp_attn->self_attn_mask.cnv; + msa_kqm = inp_attn->self_attn_mask.raw; n_kv = msa_kqm->ne[0]; n_tps = msa_kqm->ne[1]; // tokens per stream ns = msa_kqm->ne[3]; // streams in this ubatch diff --git a/src/models/mistral-large4.cpp b/src/models/mistral-large4.cpp index dbd1907a4b61..a196e6d5bc6a 100644 --- a/src/models/mistral-large4.cpp +++ b/src/models/mistral-large4.cpp @@ -289,7 +289,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr ggml_tensor * indexer_score = nullptr; if (cparams.fused_lid) { - indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.cnv); + indexer_score = ggml_lightning_indexer(ctx0, indexer_q, indexer_k, indexer_weights, inp_attn_dsa->self_attn_mask_lid.raw); cb(indexer_score, "indexer_score", il); res->add_fused_node({LLM_FUSED_OP_LIGHTNING_INDEXER, indexer_score, il}); } else { @@ -323,7 +323,7 @@ llama_model_mistral_large4::graph::graph(const llama_model & model, const llm_gr cb(indexer_score, "indexer_score", il); // mask indexer scores - ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.cnv; + ggml_tensor * indexer_kq_mask = inp_attn_dsa->self_attn_mask_lid.raw; indexer_score = ggml_add(ctx0, indexer_score, indexer_kq_mask); cb(indexer_score, "indexer_score", il); } From 37b0f9c36ae910c5a445b3934e840084ebc9b132 Mon Sep 17 00:00:00 2001 From: Dongsheng Luo Date: Tue, 3 Nov 2026 14:58:56 +0800 Subject: [PATCH 011/286] vulkan: fix out-of-bounds NVFP4 UE4M3 scale lookup (#31079) * vulkan : fix out-of-bounds NVFP4 UE4M3 scale lookup On the non-OCP path, ue4m3_fp32_lut has 128 entries and is indexed with the raw scale byte. ggml_validate_row_data accepts every byte and ggml_ue4m3_to_fp32 ignores bit 7, so a byte >= 0x80 reads past the shared array. Vulkan robustness does not cover shared memory. Size the table, both init loops, and the two host shared-memory budgets to 256 entries. ue4m3_to_fp32_build() already masks the exponent and treats only 0 and 0x7F as special, so entries 128-255 match the CPU. A 256-entry table keeps 0xFF distinct from 0x7F: the CPU decodes 0xFF as a non-zero value and 0x7F as zero. Each NVFP4 pipeline uses 512 more bytes of shared memory. ggml_fp32_to_ue4m3 never sets bit 7, so models quantized by llama.cpp are unaffected. Assisted-by: Claude Assisted-by: Codex * vulkan : fix out-of-bounds NVFP4 UE4M3 scale lookup Assisted-by: Codex --------- Co-authored-by: Dongsheng Luo <7137760+flyingdoog@users.noreply.github.com> --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 6 +++--- ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp | 2 +- ggml/src/ggml-vulkan/vulkan-shaders/types.glsl | 7 ++++--- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index ba8578716ff1..2e886c9be75f 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -1457,8 +1457,8 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec lut_size = 4*16; break; case GGML_TYPE_NVFP4: - // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[128] (types.glsl, DATA_A_NVFP4). - lut_size = 4*16 + 128u * (uint32_t)sizeof(float); + // Same kvalues budget as MXFP4 plus ue4m3_fp32_lut[256] (types.glsl, DATA_A_NVFP4). + lut_size = 4*16 + 256u * (uint32_t)sizeof(float); break; default: break; @@ -1592,7 +1592,7 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const total += 16u * (uint32_t)sizeof(int8_t); // cm1_kvalues[16] } if (src0_type == GGML_TYPE_NVFP4 && !device->ocp_fp4) { - total += 128u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[128] + total += 256u * (uint32_t)sizeof(float); // ue4m3_fp32_lut[256] } if (mul_mat_id) { total += BN * 2u * (uint32_t)sizeof(uint16_t); // row_ids[BN] (u16vec2) diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp index 17280d17b417..eaf7eb65d4e2 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp @@ -173,7 +173,7 @@ void main() { cm1_kvalues[gl_LocalInvocationIndex] = kvalues_mxfp4_const[gl_LocalInvocationIndex]; } #if !defined(USE_OCP_FP4) - for (uint i = gl_LocalInvocationIndex; i < 128u; i += BLOCK_SIZE) { + for (uint i = gl_LocalInvocationIndex; i < 256u; i += BLOCK_SIZE) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl index ec0a80c7e7d7..99ce1458a38b 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl +++ b/ggml/src/ggml-vulkan/vulkan-shaders/types.glsl @@ -1921,8 +1921,9 @@ shared int8_t kvalues_mxfp4[16]; #endif #if defined(DATA_A_NVFP4) && !defined(USE_OCP_FP4) -// UE4M3 scale in NVFP4 blocks use only 7 bits; sign (bit 7) is always zero. -shared float ue4m3_fp32_lut[128]; +// Cover all scale bytes accepted by ggml_validate_row_data. +// ue4m3_to_fp32_build() masks the exponent and mantissa, but checks the full byte for 0 and 0x7F. +shared float ue4m3_fp32_lut[256]; float ue4m3_to_fp32_build(uint u) { if (u == 0u || u == 127u) { @@ -1947,7 +1948,7 @@ void init_iq_shmem(uvec3 wgsize) kvalues_mxfp4[i] = kvalues_mxfp4_const[i]; } #if defined(DATA_A_NVFP4) - for (uint i = gl_LocalInvocationIndex.x; i < 128u; i += wgsize.x) { + for (uint i = gl_LocalInvocationIndex.x; i < 256u; i += wgsize.x) { ue4m3_fp32_lut[i] = ue4m3_to_fp32_build(i); } #endif From b74298c109c107494b792c30af4bb8fa0ce1eb2f Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 15:28:35 +0800 Subject: [PATCH 012/286] delta_net_base: refactor conv state (#30878) * delta_net: access conv state via views * fix sm tensor * delta_net: fill the conv slot index on the host The rollback slot index of the conv windows is a constant per graph shape. It is now an I32 input of the recurrent memory filled on the host, instead of an arange and an argsort computed on the device, and the meta backend no longer needs a mirrored split for ARANGE. --------- Co-authored-by: Pascal --- ggml/src/ggml-backend-meta.cpp | 20 +++++++++++++++ src/llama-graph.cpp | 19 ++++++++++++++ src/llama-graph.h | 3 +++ src/models/delta-net-base.cpp | 47 ++++++++++++++++++++-------------- 4 files changed, 70 insertions(+), 19 deletions(-) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index 2641e1f3a87e..c30bdae47f7a 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -700,6 +700,26 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( } GGML_ABORT("fatal error"); } + // permuted view: split the view dim that spans the src split axis, other dims must be inside one element of it or outside the span + if (ggml_is_permuted(tensor) && !ggml_is_permuted(tensor->src[0]) && axis >= 0 && axis < GGML_MAX_DIMS-1) { + const size_t nb_axis = tensor->src[0]->nb[axis]; + const size_t nb_next = tensor->src[0]->nb[axis+1]; + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (tensor->ne[dim] == 1 || tensor->nb[dim] < nb_axis || tensor->nb[dim]*tensor->ne[dim] != nb_next) { + continue; + } + bool others_ok = true; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i == dim || tensor->ne[i] == 1) { + continue; + } + others_ok = others_ok && (tensor->nb[i]*tensor->ne[i] <= tensor->nb[dim] || tensor->nb[i] % nb_next == 0); + } + if (others_ok) { + return {ggml_backend_meta_split_axis(dim), {0}, {1}, 1}; + } + } + } if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { return src_ss[0]; } diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index f0463f849f1b..edf1ca71f94d 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -481,6 +481,17 @@ void llm_graph_input_cls::set_input(const llama_ubatch * ubatch) { } } +static void set_input_s_slot_rev(ggml_tensor * s_slot_rev) { + if (s_slot_rev) { + GGML_ASSERT(ggml_backend_buffer_is_host(s_slot_rev->buffer)); + int32_t * data = (int32_t *) s_slot_rev->data; + + for (int64_t i = 0; i < s_slot_rev->ne[0]; ++i) { + data[i] = (int32_t) (s_slot_rev->ne[0] - 1 - i); + } + } +} + void llm_graph_input_rs::set_input(const llama_ubatch * ubatch) { GGML_UNUSED(ubatch); @@ -495,6 +506,8 @@ void llm_graph_input_rs::set_input(const llama_ubatch * ubatch) { data[i] = mctx->s_copy(i); } } + + set_input_s_slot_rev(s_slot_rev); } bool llm_graph_input_rs::can_reuse(const llm_graph_params & params) { @@ -1209,6 +1222,8 @@ void llm_graph_input_mem_hybrid::set_input(const llama_ubatch * ubatch) { data[i] = mctx->get_recr()->s_copy(i); } } + + set_input_s_slot_rev(inp_rs->s_slot_rev); } bool llm_graph_input_mem_hybrid::can_reuse(const llm_graph_params & params) { @@ -1255,6 +1270,8 @@ void llm_graph_input_mem_hybrid_k::set_input(const llama_ubatch * ubatch) { data[i] = mctx->get_recr()->s_copy(i); } } + + set_input_s_slot_rev(inp_rs->s_slot_rev); } bool llm_graph_input_mem_hybrid_k::can_reuse(const llm_graph_params & params) { @@ -1312,6 +1329,8 @@ void llm_graph_input_mem_hybrid_iswa::set_input(const llama_ubatch * ubatch) { data[i] = mctx->get_recr()->s_copy(i); } } + + set_input_s_slot_rev(inp_rs->s_slot_rev); } bool llm_graph_input_mem_hybrid_iswa::can_reuse(const llm_graph_params & params) { diff --git a/src/llama-graph.h b/src/llama-graph.h index ea2cf9f873cf..e1804fb3dec3 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -309,6 +309,9 @@ class llm_graph_input_rs : public llm_graph_input_i { ggml_tensor * s_copy_main; // I32 [n_seqs] ggml_tensor * s_copy_tail; // I32 [n_rs - 1] + // I32 [n_slot] = {n_slot - 1, ..., 0}, rollback slot of each conv window, created by the graphs that write them + ggml_tensor * s_slot_rev = nullptr; + const llama_memory_recurrent_context * mctx; // used in view offsets, need to match for valid graph reuse diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index 2e15777ee156..991b55ffb708 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -496,26 +496,35 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( // this logic assumes that the last (n_rs_seq + 1) tokens of a sequence in a batch are inside // the same ubatch, which `split_equal()` guarantees via its n_keep_tail argument - const int64_t K = (int64_t) cparams.n_rs_seq + 1; - - for (int64_t t = 1; t <= K; ++t) { - const int64_t s_idx = std::max(0, conv_input->ne[0] - conv_states->ne[0] - K + t); - const int64_t s_slot = K - t; - - ggml_tensor * conv_state_last = - ggml_view_3d(ctx0, conv_input, - conv_kernel_size - 1, conv_channels, n_seqs, - conv_input->nb[1], conv_input->nb[2], - ggml_row_size(conv_input->type, s_idx)); - - ggml_tensor * conv_state_update = - ggml_view_2d(ctx0, - conv_states_all, row_count, n_seqs, - conv_states_all->nb[1], - (s_slot * mem_size + kv_head) * row_size); - - ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_state_last, conv_state_update)); + const int64_t K = (int64_t) cparams.n_rs_seq + 1; + const int64_t n = conv_input->ne[0] - conv_states->ne[0]; // n_seq_tokens + const int64_t Kw = std::min(n + 1, K); // windows that exist in this ubatch + + // {Kw-1, ..., 0}, filled on the host and shared by every layer of the graph + if (inp->s_slot_rev == nullptr) { + inp->s_slot_rev = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, Kw); + ggml_set_input(inp->s_slot_rev); + ggml_set_name(inp->s_slot_rev, "rs_s_slot_rev"); } + + // window w starts at column n - (Kw - 1) + w (ascending time) and is rollback slot Kw - 1 - w + // slots reaching back past this ubatch are left unwritten, like the ssm snapshots in build_recurrent_attn + // the windows overlap, so the view is created with one window and widened after the size check + ggml_tensor * windows = ggml_view_4d(ctx0, conv_input, + conv_kernel_size - 1, 1, conv_channels, n_seqs, + ggml_element_size(conv_input), conv_input->nb[1], conv_input->nb[2], + ggml_row_size(conv_input->type, n - (Kw - 1))); + windows->ne[1] = Kw; + + // [d-1, slot, channel, seq] view of the cells of this ubatch + ggml_tensor * slots = ggml_view_4d(ctx0, conv_states_all, + conv_kernel_size - 1, K, conv_channels, n_seqs, + (size_t) mem_size * row_size, + ggml_row_size(conv_states_all->type, conv_kernel_size - 1), + conv_states_all->nb[1], + kv_head * row_size); + + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, slots, windows, inp->s_slot_rev)); } return conv_input; From fac3a894fc04901c3e5abaff457e024c6b52e04f Mon Sep 17 00:00:00 2001 From: FamousMonsterr Date: Tue, 3 Nov 2026 10:53:04 +0300 Subject: [PATCH 013/286] vulkan: skip maxStorageBufferRange pre-check for non-compute tensors (#31114) * vulkan: skip maxStorageBufferRange pre-check for non-compute tensors Pre-allocated leaf tensors and views (GGML_OP_NONE / GGML_OP_VIEW) are never executed by the backend, but were still rejected by the descriptor-size pre-check in ggml_backend_vk_device_supports_op when larger than maxStorageBufferRange on devices without shader_64b_indexing. The scheduler then aborts with 'pre-allocated tensor (...) in a buffer (Vulkan0) that cannot run the operation (NONE)' - e.g. the MoE expert cache banks (llama-moe-cache) of a 256-expert model with --moe-cache-mib on GCN GPUs, where a whole bank (2.4 GB) exceeds the limit while the consuming ops only access small per-step views. Skip the size pre-check for non-compute ops (NONE and VIEW); the buffer type allocation limit still bounds the actual allocation, and compute ops keep validating their sources. Tested on AMD Radeon Pro VII (gfx906, no shader_64b_indexing): - before: GGML_ABORT 'pre-allocated tensor (moe_cache.0.blk.0.ffn_down_exps.weight) ... cannot run the operation (NONE)' with llama-cli -ngl 99 --cpu-moe --moe-cache-mib 6144 (Cyber-Tiel-Coder-35B-A3B, qwen35moe, 256 experts per layer) - after: loads and generates at 8.1 t/s (vs 0.6 t/s without the cache) - regression: -ncmoe 16 path unchanged (19.2 t/s), qwen3moe/qwen35moe without cache unchanged Fixes #30697 * vulkan: use ggml_op_is_empty for the non-compute size-check skip --------- Co-authored-by: FamousMonsterr --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 2e886c9be75f..89f0470cb292 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -15550,13 +15550,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm } return true; }; - // reject any tensors larger than the max buffer size + // non-compute ops are never executed here; their size is bounded by the buffer allocation limit + const bool non_compute = ggml_op_is_empty(op->op); for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] && !tensor_size_supported(ggml_nbytes(op->src[i]))) { + if (op->src[i] && !non_compute && !tensor_size_supported(ggml_nbytes(op->src[i]))) { return false; } } - if (!tensor_size_supported(ggml_nbytes(op))) { + if (!non_compute && !tensor_size_supported(ggml_nbytes(op))) { return false; } From 36ee3caf7aa2dcb0210bedab1b61ee9f269765d3 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Tue, 3 Nov 2026 10:41:24 +0200 Subject: [PATCH 014/286] metal : fix cpy kernel for dst non-contiguous in dim 0 (#31581) kernel_cpy_t_t indexed the dst innermost dim with an implicit element-size stride instead of args.nb0, so cpy into a permuted view (nb0 != type size) wrote the wrong addresses. caught by the new test-backend-ops cpy case with permute_dst = {1, 0, 2, 3} from #31446. addressing is unchanged when dst is contiguous in dim 0. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) --- ggml/src/ggml-metal/kernels/quantize.metal | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-metal/kernels/quantize.metal b/ggml/src/ggml-metal/kernels/quantize.metal index 73679b9ab76f..663e47d5fd2f 100644 --- a/ggml/src/ggml-metal/kernels/quantize.metal +++ b/ggml/src/ggml-metal/kernels/quantize.metal @@ -26,11 +26,11 @@ kernel void kernel_cpy_t_t( const int32_t i1 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0)/args.ne0; const int32_t i0 = (n - i3*args.ne2*args.ne1*args.ne0 - i2*args.ne1*args.ne0 - i1*args.ne0); - device T1 * dst_data = (device T1 *) (dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0); + device char * dst_data = dst + i3*args.nb3 + i2*args.nb2 + i1*args.nb1 + i0*args.nb0; for (int32_t i00 = iw0*ntg[0] + tpitg.x; i00 < args.ne00;) { device const T0 * src = (device T0 *)(src0 + i03*args.nb03 + i02*args.nb02 + i01*args.nb01 + i00*args.nb00); - dst_data[i00] = (T1) src[0]; + *(device T1 *)(dst_data + i00*args.nb0) = (T1) src[0]; break; } } From 259d91709d60945911bedb9bcfb2957c6e153ea7 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Tue, 3 Nov 2026 16:51:10 +0800 Subject: [PATCH 015/286] meta: support GLM-5.3 next sm tensor (#30499) * meta: support GLM-5.3 next sm tensor * glm5-next : cont q_absorbed for -sm tensor with non-unified KV * meta : keep GGML_OP_ARANGE in ggml.h order * glm5-next : split MLA heads in pairs for -sm tensor The MLA cache has a single KV head, so the heads on a device are its GQA ratio. The HIP tile FA kernel for D=512 needs an even GQA ratio; with 3 devices the 16 heads of the test model were split 5/5/6 and it aborted. * meta : keep the split for views inside one slab of the split axis With a non-unified KV cache, build_attn_mha splits q into streams with a view. For a permuted q, e.g. the absorbed query of glm5-next, that view aborted with "view of permuted tensor not implemented". The split is now kept if every other dim of the source and the view stays inside one slab of the split axis. The per-device strides are only rescaled above it. From tetherto/qvac-fabric-llm.cpp#353. Co-authored-by: Pratik Narola * glm5-next : drop the q_absorbed cont The meta backend can now split the per-stream view of the permuted q, so the cont is not needed for -sm tensor with a non-unified KV cache. With the cont, the q of flash attention was not contiguous, and when FA ran on the CPU (WebGPU) the meta get_tensor aborted. --------- Co-authored-by: Pratik Narola --- ggml/src/ggml-backend-meta.cpp | 45 ++++++++++++++++++++++-- src/llama-arch.cpp | 1 - src/llama-model.cpp | 63 +++++++++++++++++++++++++++++++++- tests/test-llama-archs.cpp | 10 +++++- 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cpp index c30bdae47f7a..aba004646850 100644 --- a/ggml/src/ggml-backend-meta.cpp +++ b/ggml/src/ggml-backend-meta.cpp @@ -489,6 +489,32 @@ static struct ggml_tensor * ggml_backend_meta_buffer_simple_tensor(const struct return it->second[index]; } +// bytes spanned by all dims except skip_dim +static size_t ggml_backend_meta_inner_extent(const struct ggml_tensor * tensor, int skip_dim) { + size_t extent = 0; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (i != skip_dim && tensor->ne[i] > 1) { + extent += (size_t) (tensor->ne[i] - 1) * tensor->nb[i]; + } + } + return extent; +} + +// the view dim that holds the split axis of src, or -1 +// all other dims of src and view must stay inside one slab of that axis +static int ggml_backend_meta_view_slab_dim(const struct ggml_tensor * view, const struct ggml_tensor * src, int axis) { + const size_t slab = src->nb[axis]; + if (ggml_backend_meta_inner_extent(src, axis) >= slab) { + return -1; + } + for (int dim = 0; dim < GGML_MAX_DIMS; dim++) { + if (view->ne[dim] == src->ne[axis] && view->nb[dim] == slab) { + return ggml_backend_meta_inner_extent(view, dim) < slab ? dim : -1; + } + } + return -1; +} + static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(const struct ggml_tensor * tensor, bool assume_sync); static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( @@ -723,6 +749,16 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( if (src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_MIRRORED || src_ss[0].axis == GGML_BACKEND_SPLIT_AXIS_PARTIAL) { return src_ss[0]; } + // a view that only regroups dims inside one slab of the split axis keeps the split, + // the per-device strides are only rescaled above nb[axis] (e.g. the KV streams of a permuted q) + if (axis >= 0 && axis < GGML_MAX_DIMS && tensor->view_offs == tensor->src[0]->view_offs) { + const int dim = ggml_backend_meta_view_slab_dim(tensor, tensor->src[0], axis); + if (dim >= 0) { + ggml_backend_meta_split_state ret = src_ss[0]; + ret.axis = ggml_backend_meta_split_axis(dim); + return ret; + } + } GGML_ABORT("view of permuted tensor not implemented"); //return {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1}; }; @@ -1020,8 +1056,13 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state( split_state = handle_pad(src_ss); } break; case GGML_OP_PAD_REFLECT_1D: - case GGML_OP_ROLL: - case GGML_OP_ARANGE: + case GGML_OP_ROLL: { + split_state = handle_generic(src_ss, /*scalar_only =*/ true); + } break; + case GGML_OP_ARANGE: { + // no sources, every device computes the full range + split_state = {GGML_BACKEND_SPLIT_AXIS_MIRRORED, {0}, {1}, 1}; + } break; case GGML_OP_TIMESTEP_EMBEDDING: { split_state = handle_generic(src_ss, /*scalar_only =*/ true); } break; diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 40f608371c54..525f91031c0c 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -1390,7 +1390,6 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) { case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: case LLM_ARCH_KIMI_K3: - case LLM_ARCH_GLM5_NEXT: case LLM_ARCH_QWEN3TTS: case LLM_ARCH_LIMITE: return false; diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 7529f55c2025..87893bf671b6 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -410,6 +410,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str const std::string tensor_name = tensor->name; const bool is_dsv4 = ud->model->arch == LLM_ARCH_DEEPSEEK4 || ud->model->arch == LLM_ARCH_DEEPSEEK41 || (ud->model->arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_mult > 0); + const bool is_glm5_next = ud->model->arch == LLM_ARCH_GLM5_NEXT; static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight"); static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight"); @@ -429,6 +430,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_attn_out_a_weight("blk\\.\\d*\\.attn_output_a\\.weight"); static const std::regex pattern_attn_out_b_weight("blk\\.\\d*\\.attn_output_b\\.weight"); static const std::regex pattern_attn_q_b_weight ("blk\\.\\d*\\.attn_q_b\\.weight"); + static const std::regex pattern_attn_kv_b_weight("blk\\.\\d*\\.attn_(k|v)_b\\.weight"); static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight"); static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias"); @@ -441,6 +443,8 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str static const std::regex pattern_s_cache ("cache_s_l\\d*"); static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight"); static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight"); + static const std::regex pattern_kda_conv1d ("blk\\.\\d*\\.ssm_conv1d_(q|k|v)\\.weight"); + static const std::regex pattern_kda_gate_b ("blk\\.\\d*\\.ssm_(f|g)_b\\.weight"); static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight"); static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias"); @@ -532,6 +536,25 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str if (std::regex_match(tensor_name, pattern_attn_out_b_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0); } + } + if (is_glm5_next) { + // split the KDA and MLA heads, the MLA latent cache and the indexer stay mirrored + if (std::regex_match(tensor_name, pattern_kv_cache)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED); + } + if (std::regex_match(tensor_name, pattern_attn_q_b_weight) || std::regex_match(tensor_name, pattern_kda_gate_b) || + std::regex_match(tensor_name, pattern_ssm_beta)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight"); + } + if (std::regex_match(tensor_name, pattern_attn_kv_b_weight) || std::regex_match(tensor_name, pattern_kda_conv1d)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_2, "attn_output.weight"); + } + if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) || + std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) { + return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight"); + } + } + if (is_dsv4 || is_glm5_next) { if (std::regex_match(tensor_name, pattern_ffn_up_shexp_weight) || std::regex_match(tensor_name, pattern_ffn_gate_shexp_weight)) { return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down_shexp.weight"); @@ -645,6 +668,12 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str // if a model has fused tensors they need to be separated into "segments", see the comments on ggml_backend_meta_split_state struct auto get_split_segments = [&](int axis, uint32_t il) -> std::vector> { + if (is_glm5_next && std::regex_match(tensor_name, pattern_r_cache)) { + // the conv state holds the q, k and v windows one after the other + const int64_t conv_state_size = (hparams.ssm_d_conv - 1) * hparams.n_head(il) * hparams.n_embd_head_kda; + GGML_ASSERT(tensor->ne[axis] == 3*conv_state_size); + return {{conv_state_size, 3}}; + } // TODO: deduplicate condition [TAG_SPLIT_QGATE_QWEN] if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE || ud->model->arch == LLM_ARCH_CLEF || ud->model->arch == LLM_ARCH_CLOVER_MOE || ud->model->arch == LLM_ARCH_QWEN4EXP) { @@ -736,7 +765,39 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector> & segments) -> std::vector { // for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used - if (hparams.is_recr(il)) { + if (is_glm5_next && hparams.is_recr(il)) { + // KDA, each device holds whole heads + const int64_t head_dim = hparams.n_embd_head_kda; + const int64_t blck_size_perf = std::lcm(blck_size, 128); + const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim); + if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight) || + std::regex_match(tensor_name, pattern_kda_conv1d) || std::regex_match(tensor_name, pattern_kda_gate_b) || + std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_attn_out_weight)) { + return {granularity_qkv}; + } + if (std::regex_match(tensor_name, pattern_ssm_a) || std::regex_match(tensor_name, pattern_ssm_beta)) { + return {granularity_qkv / head_dim}; + } + if (std::regex_match(tensor_name, pattern_r_cache)) { + return std::vector(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1)); + } + if (std::regex_match(tensor_name, pattern_s_cache)) { + return {granularity_qkv * head_dim}; + } + } else if (is_glm5_next) { + // MLA, each device holds whole heads + // an even number of heads per device, the HIP tile FA kernel for D=512 needs an even GQA ratio + const int64_t granularity_head = std::lcm(std::lcm(hparams.n_embd_head_v_mla(), blck_size) / hparams.n_embd_head_v_mla(), int64_t(2)); + if (std::regex_match(tensor_name, pattern_attn_q_b_weight)) { + return {granularity_head * hparams.n_embd_head_k_mla()}; + } + if (std::regex_match(tensor_name, pattern_attn_kv_b_weight)) { + return {granularity_head}; + } + if (std::regex_match(tensor_name, pattern_attn_out_weight)) { + return {granularity_head * hparams.n_embd_head_v_mla()}; + } + } else if (hparams.is_recr(il)) { // linear attention const int64_t head_dim = hparams.ssm_d_state; const int64_t blck_size_perf = std::lcm(blck_size, 128); diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 0e61ebab5f4f..32569dc86c70 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -279,7 +279,6 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { || arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 - || arch == LLM_ARCH_GLM5_NEXT || arch == LLM_ARCH_MISTRAL4 || arch == LLM_ARCH_MISTRAL_LARGE4 || arch == LLM_ARCH_HY_V4) { @@ -289,6 +288,11 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { if (arch == LLM_ARCH_MISTRAL_LARGE4) { n_layer = 4; // cover NoPE, shared indexer and KV sharing layers } + } else if (arch == LLM_ARCH_GLM5_NEXT) { + // several heads so that -sm tensor can split them and the MLA flash attention has GQA > 1 + n_embd = 128; + n_head = 16; + n_ff = 192; } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; } else if (arch == LLM_ARCH_CHAMELEON) { @@ -357,6 +361,10 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { // GLM5 next KDA heads come from the uniform head count, only head_count_kv is per layer. if (arch == LLM_ARCH_GLM5_NEXT) { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); + // the MLA cache holds one latent head + for (uint32_t & n : n_head_per_layer) { + n = std::min(n, uint32_t(1)); + } } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer); } From 19e8a86fe87e5888f61b3aa975e8634b322bd776 Mon Sep 17 00:00:00 2001 From: Nidhi Date: Tue, 3 Nov 2026 14:41:05 +0530 Subject: [PATCH 016/286] tools : reject mismatched shards in GGUF split merge (#31579) --- tools/gguf-split/gguf-split.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/gguf-split/gguf-split.cpp b/tools/gguf-split/gguf-split.cpp index 73587b9fcbdf..962a59ba03c1 100644 --- a/tools/gguf-split/gguf-split.cpp +++ b/tools/gguf-split/gguf-split.cpp @@ -454,6 +454,17 @@ static void gguf_merge(const split_params & split_params) { ctx_ggufs.push_back(ctx_gguf); ctx_metas.push_back(ctx_meta); + // Verify that this split belongs to the same set as the first one + if (i_split > 0) { + auto key_split_no = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_NO); + auto key_split_count = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); + if (key_split_no < 0 || gguf_get_val_u16(ctx_gguf, key_split_no) != i_split || + key_split_count < 0 || gguf_get_val_u16(ctx_gguf, key_split_count) != n_split) { + fprintf(stderr, "\n%s: %s is not split %d of %d\n", __func__, split_path, i_split + 1, n_split); + exit(EXIT_FAILURE); + } + } + if (i_split == 0) { auto key_n_split = gguf_find_key(ctx_gguf, LLM_KV_SPLIT_COUNT); if (key_n_split < 0) { @@ -508,6 +519,14 @@ static void gguf_merge(const split_params & split_params) { fprintf(stderr, "\033[3Ddone\n"); } + + // Verify that no tensor is missing + auto key_n_tensors = gguf_find_key(ctx_ggufs[0], LLM_KV_SPLIT_TENSORS_COUNT); + if (key_n_tensors < 0 || gguf_get_val_i32(ctx_ggufs[0], key_n_tensors) != total_tensors) { + fprintf(stderr, "%s: splits contain %d tensors, which does not match %s\n", __func__, total_tensors, LLM_KV_SPLIT_TENSORS_COUNT); + exit(EXIT_FAILURE); + } + std::ofstream fout; if (!split_params.dry_run) { fout.open(split_params.output.c_str(), std::ios::binary); From 71efc559a00b236851f6ba00a629d776f2416d2b Mon Sep 17 00:00:00 2001 From: zyang-dev <267119621+zyang-dev@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:26:19 -0800 Subject: [PATCH 017/286] Add nccl support for multi-node tensor parallelism (#28967) * Add a backend interface for collectives across ranks running in separate processes, implement it in the CUDA backend using NCCL, and use it in the RPC server when available. * Drop ggml_backend_comm_allreduce_rank_t and reuse ggml_backend_comm_allreduce_tensor for per-rank communicators * reuse ggml_backend_cuda_comm_allreduce_nccl for per-rank communicators. Remove the helper function backend_comm_proc_address * Change ggml_backend_comm_get_unique_id_t and ggml_backend_comm_init_rank_t to more generic multi-process backend communicator interfaces * set maxCTAs in ncclConfig_t to 8 for dgx spark in ggml_backend_cuda_comm_init_rank --- ggml/include/ggml-backend.h | 12 ++ ggml/src/ggml-cuda/ggml-cuda.cu | 104 +++++++++++- ggml/src/ggml-rpc/ggml-rpc.cpp | 274 +++++++++++++++++++++++++------- 3 files changed, 333 insertions(+), 57 deletions(-) diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h index 735ed2c2b017..afb86338aac8 100644 --- a/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h @@ -211,6 +211,18 @@ extern "C" { // Context management and operations for faster communication between backends, used for tensor parallelism (meta backend) typedef void * (*ggml_backend_comm_init_t)(ggml_backend_t * backends, size_t n_backends); + + // Builds the communicator for the case where each rank is a separate process, so a process + // owns one backend of the communicator rather than all of them. + // + // The caller provides bootstrap_callback, which moves bytes between the ranks before the + // communicator exists. Each rank passes size bytes in send_buf, and recv_buf receives + // world_size * size bytes in rank order. The backend decides what to exchange and how many + // times to call it, and every rank must make the same calls in the same order. + // comm_init_rank blocks until all world_size ranks have joined. + typedef bool (*ggml_backend_comm_bootstrap_callback)(void * user_data, const void * send_buf, void * recv_buf, size_t size); + typedef void * (*ggml_backend_comm_init_rank_t)(ggml_backend_t backend, int rank, int world_size, ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data); + typedef void (*ggml_backend_comm_free_t)(void * comm_ctx); typedef bool (*ggml_backend_comm_allreduce_tensor_t)(void * comm_ctx, struct ggml_tensor ** tensors); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 06956ce7a3d9..84fc202a16f5 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1117,6 +1117,18 @@ struct ggml_backend_cuda_comm_context { ggml_cuda_ar_pipeline * ar_pipeline = nullptr; + // Number of ranks in the communicator when each rank is a separate process. Zero unless + // comm_init_rank created this context. + int world_size = 0; + + // Number of ranks taking part in the reduction. For a communicator from comm_init_rank this is + // world_size. Otherwise it is the number of backends this process holds. The NCCL reduction + // uses it to choose the tensor size below which it reduces in F32 rather than converting to + // BF16 first. + size_t n_ranks() const { + return world_size > 0 ? (size_t) world_size : backends.size(); + } + #ifdef GGML_USE_NCCL std::vector comms; #endif // GGML_USE_NCCL @@ -1144,6 +1156,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( } const size_t n_backends = comm_ctx->backends.size(); + const size_t n_ranks = comm_ctx->n_ranks(); for (size_t i = 0; i < n_backends; ++i) { GGML_ASSERT(tensors[i] != nullptr); @@ -1153,7 +1166,7 @@ static bool ggml_backend_cuda_comm_allreduce_nccl( // For small tensors, simply reduce them as FP32. // The following heuristic for how "small" a tensor should be is based on RTX 4090s connected via 16x PCIe 4.0. - if ((n_backends <= 2 && ne < 32768) || (n_backends == 3 && ne < 131072) || (n_backends >= 4 && ne < 262144)) { + if ((n_ranks <= 2 && ne < 32768) || (n_ranks == 3 && ne < 131072) || (n_ranks >= 4 && ne < 262144)) { for (size_t i = 0; i < n_backends; ++i) { if ((tensors[i]->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) comm_ctx->backends[i]->context; @@ -1384,6 +1397,90 @@ static void * ggml_backend_cuda_comm_init(ggml_backend_t * backends, size_t n_ba return ret; } +#ifdef GGML_USE_NCCL +// Builds the communicator with ncclCommInitRank, for the case where each rank is its own +// process. Blocks until all world_size ranks have joined. +static void * ggml_backend_cuda_comm_init_rank(ggml_backend_t backend, int rank, int world_size, + ggml_backend_comm_bootstrap_callback bootstrap_callback, void * bootstrap_callback_data) { + if (world_size < 1 || rank < 0 || rank >= world_size) { + GGML_LOG_ERROR("%s: invalid rank %d for world size %d\n", __func__, rank, world_size); + return nullptr; + } + + // Every rank sends whether its local checks passed, and rank 0 also sends the NCCL unique id. + // So all ranks either join together or fail together, and no rank waits in ncclCommInitRank alone. + struct bootstrap_msg { + uint8_t ok; + ncclUniqueId id; + }; + bootstrap_msg local; + memset(&local, 0, sizeof(local)); + local.ok = 1; + + if (!ggml_backend_is_cuda(backend)) { + GGML_LOG_ERROR("%s: backend is not a CUDA backend\n", __func__); + local.ok = 0; + } else { + // NCCL requires one distinct physical GPU per rank, see ggml_backend_cuda_comm_init_nccl. + const ggml_cuda_device_info & info = ggml_cuda_info(); + if (info.device_count > info.physical_device_count) { + GGML_LOG_ERROR("%s: CUDA virtual devices are in use, which NCCL does not support\n", __func__); + local.ok = 0; + } + } + if (local.ok && rank == 0) { + const ncclResult_t rc = ncclGetUniqueId(&local.id); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclGetUniqueId failed (%s)\n", __func__, ncclGetErrorString(rc)); + local.ok = 0; + } + } + + std::vector all(world_size); + if (!bootstrap_callback(bootstrap_callback_data, &local, all.data(), sizeof(local))) { + GGML_LOG_ERROR("%s: bootstrap exchange failed on rank %d\n", __func__, rank); + return nullptr; + } + for (int i = 0; i < world_size; i++) { + if (!all[i].ok) { + GGML_LOG_ERROR("%s: rank %d failed its setup checks\n", __func__, i); + return nullptr; + } + } + + ggml_backend_cuda_context * cuda_ctx = (ggml_backend_cuda_context *) backend->context; + + auto * ret = new ggml_backend_cuda_comm_context; + ret->backends.push_back(backend); + ret->dev_ids.push_back(cuda_ctx->device); + ret->world_size = world_size; + ret->comms.resize(1); + ret->try_allreduce = ggml_backend_cuda_comm_try_allreduce_nccl; + + // On DGX Spark, NCCL picks 64 channels by default, which is slower and uses more memory than 8. + // NCCL_MAX_CTAS still overrides this. + ncclConfig_t config = NCCL_CONFIG_INITIALIZER; + const int cc = ggml_cuda_info().devices[cuda_ctx->device].cc; + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + config.maxCTAs = 8; + } + + ggml_cuda_set_device(cuda_ctx->device); + const ncclResult_t rc = ncclCommInitRankConfig(&ret->comms[0], world_size, all[0].id, rank, &config); + if (rc != ncclSuccess) { + GGML_LOG_ERROR("%s: ncclCommInitRankConfig failed for rank %d/%d (%s)\n", + __func__, rank, world_size, ncclGetErrorString(rc)); + ret->comms.clear(); + delete ret; + return nullptr; + } + + GGML_LOG_INFO("%s: joined NCCL communicator as rank %d/%d on device %d\n", + __func__, rank, world_size, cuda_ctx->device); + return ret; +} +#endif // GGML_USE_NCCL + // Top-level dispatch -- calls the function pointer chosen by comm_init. // Returns false to let the meta-backend's butterfly run. static bool ggml_backend_cuda_comm_allreduce_tensor(void * comm_ctx_v, struct ggml_tensor ** tensors) { @@ -6384,6 +6481,11 @@ static void * ggml_backend_cuda_reg_get_proc_address(ggml_backend_reg_t reg, con if (strcmp(name, "ggml_backend_comm_allreduce_tensor") == 0) { return (void *)ggml_backend_cuda_comm_allreduce_tensor; } +#ifdef GGML_USE_NCCL + if (strcmp(name, "ggml_backend_comm_init_rank") == 0) { + return (void *)ggml_backend_cuda_comm_init_rank; + } +#endif // GGML_USE_NCCL if (strcmp(name, "ggml_backend_register_host_buffer") == 0) { return (void *)ggml_backend_cuda_register_host_buffer; } diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cpp index 5cf9339bd4bb..966bd9975d1a 100644 --- a/ggml/src/ggml-rpc/ggml-rpc.cpp +++ b/ggml/src/ggml-rpc/ggml-rpc.cpp @@ -1430,6 +1430,9 @@ class rpc_server { private: void sync_all_backends(); + void comm_free_device(uint32_t dev_id); + static bool comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors); + static bool comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size); bool get_cached_file(uint64_t hash, std::vector & data); ggml_tensor * deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor); ggml_tensor * create_node(uint64_t id, @@ -1438,15 +1441,26 @@ class rpc_server { std::unordered_map & tensor_map); - // pairwise allreduce over a direct connection to the peer server + // Per-device state for one communicator. struct comm_state { - socket_ptr peer; + // Sockets to the other servers. Rank 0 holds one per peer in rank order, so peers[i] is + // rank i + 1. Every other rank holds one to rank 0. The pairwise path reduces over peers[0]. + std::vector peers; uint32_t rank = 0; uint32_t world = 0; + ggml_backend_t pairwise_backend = nullptr; ggml_backend_buffer_ptr scratch; size_t scratch_size = 0; std::vector send_buf; std::vector recv_buf; + + // The backend's communicator. Null on the pairwise path, which has none. + void * backend_comm_ctx = nullptr; + + // The chosen reduction, set once a path is agreed. Receives backend_comm_ctx, or + // comm_state on the pairwise path. A communicator covers one device, so the caller + // passes a one-element tensor array. + ggml_backend_comm_allreduce_tensor_t allreduce_fn = nullptr; }; std::vector backends; @@ -2225,78 +2239,188 @@ void rpc_server::sync_all_backends() { } } -// The comm link between two servers uses the same caps negotiation as the client HELLO, +void rpc_server::comm_free_device(uint32_t dev_id) { + comm_state & state = comm_states[dev_id]; + if (state.backend_comm_ctx != nullptr) { + auto comm_free = (ggml_backend_comm_free_t) ggml_backend_reg_get_proc_address( + ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])), "ggml_backend_comm_free"); + if (comm_free != nullptr) { + comm_free(state.backend_comm_ctx); + } + } + state = comm_state(); +} + +// Exchange RPC transport capabilities over a peer connection. recv_first selects which side +// receives first. Both sides receiving first would deadlock. +static bool comm_caps_handshake(const socket_ptr & peer, bool recv_first) { + uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; + uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + peer->get_caps(local_caps); + if (recv_first) { + if (!peer->recv_data(remote_caps, sizeof(remote_caps)) || + !peer->send_data(local_caps, sizeof(local_caps))) { + return false; + } + } else { + if (!peer->send_data(local_caps, sizeof(local_caps)) || + !peer->recv_data(remote_caps, sizeof(remote_caps))) { + return false; + } + } + peer->update_caps(remote_caps); + return true; +} + +// The comm link between servers uses the same caps negotiation as the client HELLO, // so it gets the same transport upgrades (e.g. RDMA). bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_init_rsp & response) { response.ok = 0; - if (request.device >= backends.size() || request.world != 2 || request.rank >= request.world) { + if (request.device >= backends.size() || request.world < 2 || request.rank >= request.world) { return true; } - comm_state & state = comm_states[request.device]; - if (state.peer != nullptr) { + const uint32_t dev_id = request.device; + comm_state & state = comm_states[dev_id]; + if (state.allreduce_fn != nullptr) { response.ok = 1; return true; } - uint8_t local_caps[RPC_CONN_CAPS_SIZE] = {}; - uint8_t remote_caps[RPC_CONN_CAPS_SIZE] = {}; + + // Joining a communicator blocks until every rank arrives, so drain outstanding work first. + sync_all_backends(); + + // Rank 0 listens and accepts world-1 peers; every other rank connects back to it. if (request.rank == 0) { socket_ptr srv = socket_t::create_server("0.0.0.0", request.port); if (srv == nullptr) { LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port); return true; } - state.peer = srv->accept(); - if (state.peer == nullptr) { - return true; - } - if (!state.peer->recv_data(remote_caps, sizeof(remote_caps))) { - state.peer = nullptr; - return true; - } - state.peer->get_caps(local_caps); - if (!state.peer->send_data(local_caps, sizeof(local_caps))) { - state.peer = nullptr; - return true; + // Store peers[i] as rank i + 1, as comm_bootstrap_exchange needs. + // Each peer sends its rank because peers may connect in any order. + state.peers.resize(request.world - 1); + for (uint32_t i = 1; i < request.world; i++) { + socket_ptr peer = srv->accept(); + uint32_t peer_rank = 0; + if (peer == nullptr || !comm_caps_handshake(peer, /*recv_first =*/ true) || + !peer->recv_data(&peer_rank, sizeof(peer_rank)) || + peer_rank == 0 || peer_rank >= request.world || state.peers[peer_rank - 1] != nullptr) { + LOG_ERROR("[%s] bad peer connection (rank %u)\n", __func__, peer_rank); + state.peers.clear(); + return true; + } + state.peers[peer_rank - 1] = std::move(peer); } - state.peer->update_caps(remote_caps); } else { const std::string host(request.host, strnlen(request.host, sizeof(request.host))); + socket_ptr peer; // rank 0 may not be listening yet, retry for a few seconds - for (int i = 0; i < 100 && state.peer == nullptr; i++) { - state.peer = socket_t::connect(host.c_str(), request.port); - if (state.peer == nullptr) { + for (int i = 0; i < 100 && peer == nullptr; i++) { + peer = socket_t::connect(host.c_str(), request.port); + if (peer == nullptr) { std::this_thread::sleep_for(std::chrono::milliseconds(50)); } } - if (state.peer == nullptr) { + if (peer == nullptr) { LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port); return true; } - state.peer->get_caps(local_caps); - if (!state.peer->send_data(local_caps, sizeof(local_caps)) || - !state.peer->recv_data(remote_caps, sizeof(remote_caps))) { - state.peer = nullptr; + if (!comm_caps_handshake(peer, /*recv_first =*/ false) || + !peer->send_data(&request.rank, sizeof(request.rank)) || !peer->flush()) { return true; } - state.peer->update_caps(remote_caps); + state.peers.push_back(std::move(peer)); } + state.rank = request.rank; state.world = request.world; - LOG_INFO("[%s] device %u joined pairwise comm as rank %u\n", __func__, request.device, request.rank); + + // Forces the pairwise path, for comparing the two on one build. Setting it on any one rank + // is enough, because the negotiation spreads the decision. + const bool no_backend_communicator = std::getenv("GGML_RPC_NO_BACKEND_COMMUNICATOR") != nullptr; + + ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backends[dev_id])); + auto init_rank = (ggml_backend_comm_init_rank_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_init_rank"); + auto allreduce_fn = (ggml_backend_comm_allreduce_tensor_t) + ggml_backend_reg_get_proc_address(reg, "ggml_backend_comm_allreduce_tensor"); + + // The ranks must agree on backend communicator or pairwise. Every rank shares whether its + // backend provides a communicator, and all ranks use it only if every rank can. + const uint8_t self_can_use_backend_comm = !no_backend_communicator && init_rank != nullptr && allreduce_fn != nullptr; + std::vector all_can_use_backend_comm(request.world); + if (!comm_bootstrap_exchange(&state, &self_can_use_backend_comm, all_can_use_backend_comm.data(), + sizeof(self_can_use_backend_comm))) { + state.peers.clear(); + return true; + } + const bool use_backend_communicator = std::all_of(all_can_use_backend_comm.begin(), all_can_use_backend_comm.end(), + [](uint8_t c) { return c != 0; }); + + if (use_backend_communicator) { + void * backend_ctx = init_rank(backends[dev_id], (int) request.rank, (int) request.world, comm_bootstrap_exchange, &state); + if (backend_ctx == nullptr) { + LOG_ERROR("[%s] failed to join communicator as rank %u/%u\n", __func__, request.rank, request.world); + state.peers.clear(); + return true; + } + state.backend_comm_ctx = backend_ctx; + state.allreduce_fn = allreduce_fn; + // The sockets have done their job; the collective runs between the backends from here on. + state.peers.clear(); + LOG_INFO("[%s] device %u using backend communicator, rank %u/%u\n", __func__, dev_id, request.rank, request.world); + response.ok = 1; + return true; + } + + // No backend communicator, so fall back to the pairwise exchange, which only covers two ranks. + if (request.world != 2) { + LOG_ERROR("[%s] world size %u requires a backend communicator\n", __func__, request.world); + state.peers.clear(); + return true; + } + state.pairwise_backend = backends[dev_id]; + state.allreduce_fn = comm_allreduce_pairwise; + LOG_INFO("[%s] device %u using pairwise communicator, rank %u\n", __func__, dev_id, request.rank); response.ok = 1; return true; } -bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { - if (request.device >= backends.size()) { - return false; - } - comm_state & state = comm_states[request.device]; - if (state.peer == nullptr) { - LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); - return false; +// Bootstrap callback for ggml_backend_comm_init_rank_t, over the peer sockets. Rank 0 collects +// size bytes from every rank in rank order, then sends the whole array back to every rank. +// user_data is the comm_state. +bool rpc_server::comm_bootstrap_exchange(void * user_data, const void * send_buf, void * recv_buf, size_t size) { + comm_state & state = *(comm_state *) user_data; + uint8_t * recv = (uint8_t *) recv_buf; + const size_t total = (size_t) state.world * size; + + if (state.rank == 0) { + memcpy(recv, send_buf, size); + for (size_t i = 0; i < state.peers.size(); i++) { + if (!state.peers[i]->recv_data(recv + (i + 1) * size, size)) { + return false; + } + } + for (const socket_ptr & peer : state.peers) { + if (!peer->send_data(recv, total) || !peer->flush()) { + return false; + } + } + return true; } - ggml_backend_t backend = backends[request.device]; + return state.peers[0]->send_data(send_buf, size) && state.peers[0]->flush() && + state.peers[0]->recv_data(recv, total); +} + +// Reduces this rank's contribution in place, exchanging partials with the peer over peers[0]. +// Used when the backend provides no communicator. comm_ctx_v is the comm_state. +bool rpc_server::comm_allreduce_pairwise(void * comm_ctx_v, ggml_tensor ** tensors) { + comm_state & state = *(comm_state *) comm_ctx_v; + ggml_tensor * t_dst = tensors[0]; + ggml_backend_t backend = state.pairwise_backend; + + const size_t nbytes = ggml_nbytes(t_dst); + const int64_t ne = ggml_nelements(t_dst); size_t ctx_size = 16*ggml_tensor_overhead() + 2*ggml_graph_overhead_custom(8, false); struct ggml_init_params params = { @@ -2307,16 +2431,7 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { ggml_context_ptr ctx_ptr { ggml_init(params) }; GGML_ASSERT(ctx_ptr != nullptr); ggml_context * ctx = ctx_ptr.get(); - ggml_tensor * t_dst = deserialize_tensor(ctx, &request.tensor); - if (t_dst == nullptr || t_dst->buffer == nullptr) { - LOG_ERROR("[%s] error deserializing tensor\n", __func__); - return false; - } - const size_t nbytes = ggml_nbytes(t_dst); - const int64_t ne = ggml_nelements(t_dst); - if (nbytes == 0) { - return true; - } + // reduce large partials in bf16 to halve the wire bytes; small (decode-sized) ones // stay f32 since the extra casts and sync cost more than the bytes saved const bool wire_bf16 = t_dst->type == GGML_TYPE_F32 && ne >= 32768; @@ -2372,13 +2487,13 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { // rank 0 sends first, rank 1 receives first, so large payloads cannot deadlock if (state.rank == 0) { - if (!state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush() || - !state.peer->recv_data(state.recv_buf.data(), wire_bytes)) { + if (!state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush() || + !state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes)) { return false; } } else { - if (!state.peer->recv_data(state.recv_buf.data(), wire_bytes) || - !state.peer->send_data(state.send_buf.data(), wire_bytes) || !state.peer->flush()) { + if (!state.peers[0]->recv_data(state.recv_buf.data(), wire_bytes) || + !state.peers[0]->send_data(state.send_buf.data(), wire_bytes) || !state.peers[0]->flush()) { return false; } } @@ -2408,11 +2523,50 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { return true; } +bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) { + if (request.device >= backends.size()) { + return false; + } + comm_state & state = comm_states[request.device]; + if (state.allreduce_fn == nullptr) { + LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device); + return false; + } + + struct ggml_init_params params = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ NULL, + /*.no_alloc =*/ true, + }; + ggml_context_ptr ctx_ptr { ggml_init(params) }; + GGML_ASSERT(ctx_ptr != nullptr); + ggml_tensor * t_dst = deserialize_tensor(ctx_ptr.get(), &request.tensor); + if (t_dst == nullptr || t_dst->buffer == nullptr) { + LOG_ERROR("[%s] error deserializing tensor\n", __func__); + return false; + } + if (ggml_nbytes(t_dst) == 0) { + return true; + } + + // Neither path needs a wait here. The pairwise path waits internally, and the backend orders + // its reduction after the work that produced the partial sum. + void * comm_ctx = state.backend_comm_ctx != nullptr ? state.backend_comm_ctx : &state; + if (!state.allreduce_fn(comm_ctx, &t_dst)) { + LOG_ERROR("[%s] allreduce failed for tensor %s\n", __func__, t_dst->name); + return false; + } + return true; +} + bool rpc_server::comm_free(const rpc_msg_comm_free_req & request) { if (request.device >= backends.size()) { return false; } - comm_states[request.device] = comm_state(); + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + comm_free_device(request.device); return true; } @@ -2432,6 +2586,14 @@ bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request } rpc_server::~rpc_server() { + // Wait for outstanding work. Destroying a communicator while its kernels are still running + // is unsafe. + sync_all_backends(); + + // Free any communicator still open. A client can disconnect without sending COMM_FREE. + for (uint32_t dev_id = 0; dev_id < comm_states.size(); dev_id++) { + comm_free_device(dev_id); + } for (auto buffer : buffers) { ggml_backend_buffer_free(buffer); } @@ -2997,7 +3159,7 @@ static void ggml_backend_rpc_comm_free(void * comm_ctx_v) { } static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_backends) { - if (n_backends != 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { + if (n_backends < 2 || std::getenv("GGML_RPC_NO_COMM") != nullptr) { return nullptr; } std::vector ranks; From 740251a920abf92b7744d9c0518ca157db0db579 Mon Sep 17 00:00:00 2001 From: Bhargav Krish Date: Tue, 3 Nov 2026 15:31:50 +0530 Subject: [PATCH 018/286] cuda : gather batched tensor reads into mapped host memory (PR to master) (#31391) * cuda: gather batched tensor reads into mapped host memory Assisted-by: Codex * add comments Assisted-by: Codex * simplify the implementation based on checks that exist earlier in the flow * simplify batched tensor gather indexing * add conditional dispatch * address comments - make host registration raii, better test mindful of ci time, assert instead of return false * cuda : gather small reads and use pinned DMA for large reads * cuda : use std::vector for host gather descriptors * cuda : add 8-byte gather copies, generalize tensor test and log registration failures * common buffer for descriptors * simplify stream sync + increase to 128k descriptor pinned buf --- ggml/src/ggml-cuda/batch-get.cu | 198 +++++++++++++++++++++++++++++++ ggml/src/ggml-cuda/batch-get.cuh | 7 ++ ggml/src/ggml-cuda/ggml-cuda.cu | 6 + tests/test-backend-tensor.cpp | 22 ++++ 4 files changed, 233 insertions(+) create mode 100644 ggml/src/ggml-cuda/batch-get.cu create mode 100644 ggml/src/ggml-cuda/batch-get.cuh diff --git a/ggml/src/ggml-cuda/batch-get.cu b/ggml/src/ggml-cuda/batch-get.cu new file mode 100644 index 000000000000..60c4747b967c --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cu @@ -0,0 +1,198 @@ +#include "batch-get.cuh" +#include "common.cuh" +#include "ggml-cpp.h" + +#include + +#if !defined(GGML_USE_HIP) && !defined(GGML_USE_MUSA) +struct ggml_cuda_batch_get_descriptor { + const unsigned char * src; + unsigned char * dst; + size_t size; +}; + +static __global__ void ggml_cuda_batch_get_kernel(const ggml_cuda_batch_get_descriptor * copies, size_t count) { + // Blocks along y cooperate on each descriptor. + const size_t index = blockIdx.x; + if (index >= count) { + return; + } + const size_t first = (size_t) blockIdx.y * blockDim.x + threadIdx.x; + const size_t stride = (size_t) gridDim.y * blockDim.x; + const auto copy = copies[index]; + // Use 16-, 8-, 4-, or 2-byte copies when both pointers meet that alignment; threads copy separate chunks. + const uintptr_t alignment = (uintptr_t) copy.src | (uintptr_t) copy.dst; + size_t tail = 0; + if (alignment % 16 == 0) { + for (size_t i = first; i < copy.size / 16; i += stride) { + ggml_cuda_memcpy_1<16>(copy.dst + 16*i, copy.src + 16*i); + } + tail = copy.size / 16 * 16; + } else if (alignment % 8 == 0) { + for (size_t i = first; i < copy.size / 8; i += stride) { + ggml_cuda_memcpy_1<8>(copy.dst + 8*i, copy.src + 8*i); + } + tail = copy.size / 8 * 8; + } else if (alignment % 4 == 0) { + for (size_t i = first; i < copy.size / 4; i += stride) { + ggml_cuda_memcpy_1<4>(copy.dst + 4*i, copy.src + 4*i); + } + tail = copy.size / 4 * 4; + } else if (alignment % 2 == 0) { + for (size_t i = first; i < copy.size / 2; i += stride) { + ggml_cuda_memcpy_1<2>(copy.dst + 2*i, copy.src + 2*i); + } + tail = copy.size / 2 * 2; + } + // Copy trailing bytes, or the entire range when no wider alignment is available. + for (size_t i = tail + first; i < copy.size; i += stride) { + ggml_cuda_memcpy_1<1>(copy.dst + i, copy.src + i); + } +} + +struct ggml_cuda_batch_get_mapping { + void * host = nullptr; + unsigned char * device = nullptr; + + ggml_cuda_batch_get_mapping(void * host, size_t size) { + const cudaError_t err = cudaHostRegister(host, size, cudaHostRegisterMapped | cudaHostRegisterPortable); + if (err != cudaSuccess) { + (void) cudaGetLastError(); + GGML_LOG_DEBUG("%s: failed to register %.2f MiB of pinned memory: %s\n", __func__, + size / 1024.0 / 1024.0, cudaGetErrorString(err)); + return; + } + this->host = host; + if (cudaHostGetDevicePointer((void **) &device, host, 0) != cudaSuccess) { + (void) cudaGetLastError(); + device = nullptr; + return; + } + } + + ggml_cuda_batch_get_mapping(const ggml_cuda_batch_get_mapping &) = delete; + ggml_cuda_batch_get_mapping & operator=(const ggml_cuda_batch_get_mapping &) = delete; + + ~ggml_cuda_batch_get_mapping() { + if (host) { + // Release only the registration acquired by this object. + CUDA_CHECK(cudaHostUnregister(host)); + } + } +}; +#endif + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors) { +#if defined(GGML_USE_HIP) || defined(GGML_USE_MUSA) + GGML_UNUSED(device); + GGML_UNUSED(tensors); + GGML_UNUSED(data); + GGML_UNUSED(offsets); + GGML_UNUSED(sizes); + GGML_UNUSED(n_tensors); + return false; +#else + // Provisional dispatch values based on experiments. + constexpr size_t min_batch_reads = 128; + constexpr size_t max_gather_read_bytes = 128 * 1024; + // Collect non-empty CPU destination ranges. + std::vector> ranges; + ranges.reserve(n_tensors); + size_t payload_bytes = 0; + // Number of non-empty reads whose size is at most max_gather_read_bytes. + size_t n_reads_small = 0; + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] == 0) { + continue; + } + const uintptr_t address = (uintptr_t) data[i]; + GGML_ASSERT(address <= UINTPTR_MAX - sizes[i]); + GGML_ASSERT(payload_bytes <= SIZE_MAX - sizes[i]); + payload_bytes += sizes[i]; + n_reads_small += sizes[i] <= max_gather_read_bytes; + ranges.emplace_back(address, address + sizes[i]); + } + if (ranges.empty()) { + return true; + } + // Total number of non-empty reads. + const size_t n_reads = ranges.size(); + if (n_reads > SIZE_MAX / sizeof(ggml_cuda_batch_get_descriptor) || n_reads > INT_MAX) { + return false; + } + if (n_reads < 2 || (n_reads < min_batch_reads && n_reads_small == n_reads)) { + return false; + } + std::sort(ranges.begin(), ranges.end()); + for (size_t i = 1; i < ranges.size(); ++i) { + // Preserve the ordered copy path when CPU destinations overlap. + if (ranges[i].first < ranges[i - 1].second) { + return false; + } + } + // Find the CPU span to register, including gaps between destinations. + const uintptr_t begin = ranges.front().first; + const uintptr_t end = ranges.back().second; + if (end - begin - payload_bytes > payload_bytes) { + return false; + } + // Register and map the existing CPU destination so the kernel can write to it directly. + ggml_cuda_batch_get_mapping mapping((void *) begin, end - begin); + if (!mapping.host) { + return false; + } + // Reads selected for gather; zero means all reads use DMA. + size_t n_reads_gather = mapping.device && n_reads_small >= min_batch_reads ? n_reads_small : 0; + // Keep descriptor storage alive until the final stream synchronization. + constexpr size_t max_batch_descriptors = 128 * 1024; + const size_t bytes = std::min(n_reads_gather, max_batch_descriptors) * sizeof(ggml_cuda_batch_get_descriptor); + ggml_backend_buffer_ptr descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(ggml_backend_cuda_buffer_type(device), bytes) : nullptr); + if (!descriptors) { + n_reads_gather = 0; + } + const auto host_type = ggml_backend_cuda_host_buffer_type(); + ggml_backend_buffer_ptr host_descriptors(n_reads_gather ? ggml_backend_buft_alloc_buffer(host_type, bytes) : nullptr); + if (!host_descriptors || ggml_backend_buffer_get_type(host_descriptors.get()) != host_type) { + n_reads_gather = 0; + } + if (n_reads_gather) { + auto * copies = static_cast(ggml_backend_buffer_get_base(host_descriptors.get())); + auto * device_copies = static_cast(ggml_backend_buffer_get_base(descriptors.get())); + const int threads = 256; + const size_t nsm = ggml_cuda_info().devices[device].nsm; + size_t i = 0; + for (size_t first = 0; first < n_reads_gather; first += max_batch_descriptors) { + const size_t n_batch = std::min(n_reads_gather - first, max_batch_descriptors); + size_t count = 0; + for (; i < n_tensors && count < n_batch; ++i) { + if (sizes[i] == 0 || sizes[i] > max_gather_read_bytes) { + continue; + } + copies[count++] = {(const unsigned char *) tensors[i]->data + offsets[i], + mapping.device + ((uintptr_t) data[i] - begin), sizes[i]}; + } + const size_t blocks_per_copy = std::min(UINT16_MAX, std::max(1, (nsm + n_batch - 1) / n_batch)); + const dim3 blocks((unsigned int) n_batch, (unsigned int) blocks_per_copy); + // Stream ordering keeps each kernel ahead of the next descriptor overwrite. + CUDA_CHECK(cudaMemcpyAsync(device_copies, copies, n_batch * sizeof(copies[0]), cudaMemcpyHostToDevice, cudaStreamPerThread)); + ggml_cuda_batch_get_kernel<<>>(device_copies, n_batch); + CUDA_CHECK(cudaGetLastError()); + if (first + n_batch < n_reads_gather) { + // Wait before refilling the pinned host descriptor buffer. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + } + } + } + // Keep the destination pinned for DMA too; overlapping destinations already declined above. + for (size_t i = 0; i < n_tensors; ++i) { + if (sizes[i] && (!n_reads_gather || sizes[i] > max_gather_read_bytes)) { + CUDA_CHECK(cudaMemcpyAsync(data[i], (const char *) tensors[i]->data + offsets[i], sizes[i], cudaMemcpyDeviceToHost, cudaStreamPerThread)); + } + } + // Wait for GPU writes before releasing the CPU mapping and descriptor buffers. + CUDA_CHECK(cudaStreamSynchronize(cudaStreamPerThread)); + return true; +#endif +} diff --git a/ggml/src/ggml-cuda/batch-get.cuh b/ggml/src/ggml-cuda/batch-get.cuh new file mode 100644 index 000000000000..eb9bf0cc81bd --- /dev/null +++ b/ggml/src/ggml-cuda/batch-get.cuh @@ -0,0 +1,7 @@ +#pragma once + +#include "ggml-backend.h" + +bool ggml_cuda_get_tensor_n(int device, + const ggml_tensor * const * tensors, void * const * data, const size_t * offsets, + const size_t * sizes, size_t n_tensors); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 84fc202a16f5..ee28955f42b4 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -9,6 +9,7 @@ #include "ggml-cuda/arange.cuh" #include "ggml-cuda/argmax.cuh" #include "ggml-cuda/argsort.cuh" +#include "ggml-cuda/batch-get.cuh" #include "ggml-cuda/binbcast.cuh" #include "ggml-cuda/clamp.cuh" #include "ggml-cuda/col2im-1d.cuh" @@ -886,6 +887,11 @@ static void ggml_backend_cuda_buffer_get_tensor_n(ggml_backend_buffer_t buffer, ggml_cuda_set_device(ctx->device); + if (!ggml_cuda_pinned_memory_disabled() && + ggml_cuda_get_tensor_n(ctx->device, tensors, data, offsets, sizes, n_tensors)) { + return; + } + for (size_t i = 0; i < n_tensors; i++) { if (sizes[i] == 0) { continue; diff --git a/tests/test-backend-tensor.cpp b/tests/test-backend-tensor.cpp index 3e8349c0b1b9..9685bc277789 100644 --- a/tests/test-backend-tensor.cpp +++ b/tests/test-backend-tensor.cpp @@ -148,6 +148,28 @@ static bool test_tensor_n(ggml_backend_buffer_type_t buft) { } } + { + uint8_t input[24], expected[128], out[128] = {}; + for (size_t i = 0; i < sizeof(input); i++) { + input[i] = (uint8_t) (i + 1); + } + ggml_backend_tensor_set(t_big, input, 0, sizeof(input)); + + // Verify scattered byte reads, including repeated source offsets. + std::vector tensors(128, t_big); + std::vector offsets(128), sizes(128, 1), data_offsets(128); + for (size_t i = 0; i < tensors.size(); i++) { + offsets[i] = (i * 5) % sizeof(input); + data_offsets[i] = i; + expected[i] = input[offsets[i]]; + } + + ggml_backend_tensor_get_n(tensors.data(), out, offsets.data(), sizes.data(), data_offsets.data(), tensors.size()); + if (memcmp(out, expected, sizeof(out)) != 0) { + return false; + } + } + return true; } From 8fc0376721e9662d498a5706d7ee8e7574ce99f4 Mon Sep 17 00:00:00 2001 From: mmalpe-nv Date: Tue, 3 Nov 2026 15:54:42 +0530 Subject: [PATCH 019/286] cuda: potential fix for Q8_1 correction sum (#31247) * cuda: potential fix for Q8_1 correction sum * Update ggml/src/ggml-cuda/quantize.cu Co-authored-by: Gaurav Garg * addressed review comment to change the quantize_mmq_q8_1 flow as well --------- Co-authored-by: Gaurav Garg --- ggml/src/ggml-cuda/quantize.cu | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu index ea089a8c6249..727764cdd061 100644 --- a/ggml/src/ggml-cuda/quantize.cu +++ b/ggml/src/ggml-cuda/quantize.cu @@ -83,21 +83,21 @@ static __global__ void quantize_q8_1( ggml_cuda_pdl_sync(); const float xi = i0 < ne00 ? x[i03*s03 + i02*s02 + i01*s01 + i00] : 0.0f; float amax = fabsf(xi); - float sum = xi; amax = warp_reduce_max(amax); - sum = warp_reduce_sum(sum); const float d = amax / 127.0f; const int8_t q = amax == 0.0f ? 0 : roundf(xi / d); + const int sum = warp_reduce_sum((int) q); + y[ib].qs[iqs] = q; if (iqs > 0) { return; } - y[ib].ds = make_half2(d, sum); + y[ib].ds = make_half2(d, d * sum); } __device__ __forceinline__ uint8_t compute_e8m0_scale(float amax) { @@ -500,17 +500,6 @@ static __global__ void quantize_mmq_q8_1( amax = fmaxf(amax, __shfl_xor_sync(0xFFFFFFFF, amax, offset, WARP_SIZE)); } - float sum; - if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { - sum = xi.x + xi.y + xi.z + xi.w; - - // Calculate sums across vals_per_sum/4 threads. -#pragma unroll - for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { - sum += __shfl_xor_sync(0xFFFFFFFF, sum, offset, WARP_SIZE); - } - } - const float d_inv = 127.0f / amax; char4 q; q.x = roundf(xi.x*d_inv); @@ -519,6 +508,19 @@ static __global__ void quantize_mmq_q8_1( q.w = roundf(xi.w*d_inv); const float d = 1.0f / d_inv; + float sum = 0.0f; + if (ds_layout != MMQ_Q8_1_DS_LAYOUT_D4) { + int sum_q = int(q.x) + int(q.y) + int(q.z) + int(q.w); + + // Calculate sums across vals_per_sum/4 threads. +#pragma unroll + for (int offset = vals_per_sum/8; offset > 0; offset >>= 1) { + sum_q += __shfl_xor_sync(0xFFFFFFFF, sum_q, offset, WARP_SIZE); + } + + sum = d * float(sum_q); + } + // write the block once (normal) or to each of the token's compact rows (scatter) const int nwrite = scatter ? n_expert_used : 1; #pragma unroll From 0ef27309b78d054832e44a95a6fb9314d4b7363c Mon Sep 17 00:00:00 2001 From: Daniel Bevenius Date: Tue, 3 Nov 2026 02:40:51 -0800 Subject: [PATCH 020/286] conversion : use original model tensor names in _flush_nvfp4_experts (#31427) This commit updates the _flush_nvfp4_experts to take a new parameter which is a dictionary mapping the key to the original model tensor name. The motivation for this is that I noticed that the tensor names being looked up were not the original model tensor names, for example: ```console model.layers.1.mlp.experts.up_proj.weight ``` But _nvfp4_layer_algo uses the original per-expert names: ```console (Pdb) p self._nvfp4_layer_algo { 'backbone.layers.1.mixer.experts.0.up_proj': 'W4A16_NVFP4', ... ``` This means that the lookup will never match for experts and fall back to _nvfp4_global_algo. If the global algo is W4A16_NVFP4 then this would go unnoticed, but for a model like nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4 which is MIXED_PRECISION, the expert tensors will not get prec_a4=false so they would use the native W4A4 path on Blackwell even though the original model tensors declare them as W4A16_NVFP4. --- conversion/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/conversion/base.py b/conversion/base.py index 6347d7fd32a5..1c7aecec877c 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1277,6 +1277,7 @@ def _generate_nvfp4_tensors(self): expert_scales: dict[tuple[int, str], list[tuple[int, float]]] = {} expert_input_scales: dict[tuple[int, str], list[tuple[int, float]]] = {} expert_shapes: dict[tuple[int, str], list[int]] = {} + expert_names: dict[tuple[int, str], str] = {} n_experts = self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True) or 0 consumed: list[str] = [] @@ -1325,6 +1326,7 @@ def _generate_nvfp4_tensors(self): expert_scales[key] = [] expert_input_scales[key] = [] expert_shapes[key] = shape + expert_names[key] = name expert_blocks[key].append((expert_id, raw.copy())) # Collect per-expert scale2 (scalar per expert) expert_scales[key].append((expert_id, float(scale2.float().sum()))) @@ -1333,13 +1335,13 @@ def _generate_nvfp4_tensors(self): # Flush when all experts for this (layer, proj) are collected if n_experts > 0 and len(expert_blocks[key]) >= n_experts: - self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type) + self._flush_nvfp4_experts(key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type) else: self._repack_nvfp4(name, weight, scale, scale2, input_scale) # Flush any remaining experts (fallback if n_experts was unknown) for bid, proj_type in list(expert_blocks.keys()): - self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type) + self._flush_nvfp4_experts((bid, proj_type), expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type) # Remove consumed tensors so get_tensors/modify_tensors won't see them for name in consumed: @@ -1350,7 +1352,7 @@ def _generate_nvfp4_tensors(self): if name.endswith((".k_scale", ".v_scale")): del self.model_tensors[name] - def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, bid, proj_type): + def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_scales, expert_shapes, expert_names, bid, proj_type): experts = expert_blocks.pop(key) scales = expert_scales.pop(key) input_scales = expert_input_scales.pop(key) @@ -1362,7 +1364,7 @@ def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_s new_name = self.map_tensor_name(merged_name) logger.info(f"Repacked {new_name} with shape [{len(experts)}, {shape[0]}, {shape[1]}] and quantization NVFP4") self.gguf_writer.add_tensor(new_name, merged, raw_dtype=gguf.GGMLQuantizationType.NVFP4) - self._tag_prec_a4(merged_name, new_name) + self._tag_prec_a4(expert_names.pop(key), new_name) scales.sort(key=lambda x: x[0]) self._write_scales_tensor(new_name.replace(".weight", ".scale"), [s[1] for s in scales]) From 0978a6e9022fb9bd6874b86154ee304e81929b9f Mon Sep 17 00:00:00 2001 From: Jasmine-tim <403568753@qq.com> Date: Tue, 3 Nov 2026 19:26:29 +0800 Subject: [PATCH 021/286] ggml: avoid signed overflow in ggml_roll bounds check (#30375) ggml_roll validated each shift with abs(shift), but abs(INT_MIN) is signed integer overflow UB evaluated before the bounds check runs. Compare with a signed-safe test instead. --- ggml/src/ggml.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 3d60c44e594f..b91d98845826 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -5389,10 +5389,10 @@ struct ggml_tensor * ggml_roll( int shift2, int shift3) { GGML_ASSERT(a->nb[0] == ggml_type_size(a->type)); - GGML_ASSERT(abs(shift0) < a->ne[0]); - GGML_ASSERT(abs(shift1) < a->ne[1]); - GGML_ASSERT(abs(shift2) < a->ne[2]); - GGML_ASSERT(abs(shift3) < a->ne[3]); + GGML_ASSERT(shift0 > -a->ne[0] && shift0 < a->ne[0]); + GGML_ASSERT(shift1 > -a->ne[1] && shift1 < a->ne[1]); + GGML_ASSERT(shift2 > -a->ne[2] && shift2 < a->ne[2]); + GGML_ASSERT(shift3 > -a->ne[3] && shift3 < a->ne[3]); struct ggml_tensor * result = ggml_dup_tensor(ctx, a); From f9448cddc48c129a4abc7ff9c2a84c2b4c101c61 Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Tue, 3 Nov 2026 12:29:10 +0100 Subject: [PATCH 022/286] model : record layer inputs in LFM3 for spec dec (#31598) --- src/models/lfm3.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp index 174ce9cf7e38..b74dea41b0b0 100644 --- a/src/models/lfm3.cpp +++ b/src/models/lfm3.cpp @@ -157,6 +157,8 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr ggml_tensor * inp_out_ids = build_inp_out_ids(); for (int il = 0; il < n_layer; ++il) { + res->t_layer_inp[il] = cur; + auto * prev_cur = cur; cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); From 0943ca5472f32774375c1648912357b20f57a1c6 Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 3 Nov 2026 12:30:49 +0100 Subject: [PATCH 023/286] ui: add Playwright harness for recording demo videos (#31318) * ui: add Playwright harness for recording demo videos Scripted walkthroughs under tools/ui/demo are recorded with Playwright and exported as H.264 mp4 for pull requests (demo/pr) and docs (demo/docs). Fixture MCP servers and seeded settings keep recordings deterministic, a drawn cursor and captions make the flow readable, and DEMO_UI_DIR points the recording at another checkout such as a PR worktree. Assisted-by: Claude (Opus 5.5 Medium) * ui: record demo scripts from PR descriptions and a configurable storage Demo scripts no longer live in the repo. `npm run demo:pr -- ` records the `ts demo` block from a PR description against the PR head, uploads the mp4, a GIF preview and the script to DEMO_STORAGE, and links the preview in a `## Demo` section of the PR description. `npm run demo:docs` records the docs scripts kept in DEMO_STORAGE. DEMO_STORAGE (hf:// buckets or file://) and the public URL are read from tools/ui/.env. Assisted-by: Claude (Opus 5.5 Medium) * ui: import generated conversations into demo recordings Demos can now start with existing chats: `test.use({ traces })` imports llama.app JSONL traces through Settings > Import/Export before the video starts. Built-in traces are generated deterministically, other names are downloaded from DEMO_TRACES_DATASET, and `demo:traces` pushes the built-in ones there. Adds `demo.scroll` for wheel scrolling. Assisted-by: Claude (Opus 5.5 Medium) * ui: keep demo captions clear of the chat form and isolate concurrent runs Captions move to the top of the frame, since at the bottom they covered the chat form a demo is often about. Each recording run gets its own Playwright output folder, so a run starting elsewhere no longer clears another run's video before it is saved. Assisted-by: Claude (Opus 5.5 Medium) * ui: record demos at 2x and 60 fps with human-like timing Recording goes through Chromium's screencast instead of Playwright's video, which encoded VP8 at about 1 Mbit/s at 1x. Frames are captured at the device pixel ratio (DEMO_SCALE, default 2) and encoded into a 60 fps H.264 mp4, starting once the page has settled. The cursor now glides faster along a slight arc with Fitts-like durations, settles briefly before each click, and typing and pauses vary a little. The randomness is seeded per demo, so re-recording gives the same motion. Assisted-by: Claude (Opus 5.5 Medium) * ui: document the demo harness prerequisites The demo README now lists the tools the harness needs and what each is for, and walks through creating the Hugging Face bucket and the optional traces dataset, with the matching .env settings. Assisted-by: Claude (Opus 5.5 Medium) * ui: version linked demo URLs so re-recorded previews refresh GitHub's image proxy caches a preview by URL, so a demo re-recorded at the same PR head kept showing the old GIF. The Demo section now links the files with a content hash in the query string. Assisted-by: Claude (Opus 5.5 Medium) * ui: label PRs that feature a video demo Adds a `video demo` label and a workflow that applies it when a PR description has a demo written by the UI demo pipeline (the managed demo:start/demo:end markers, or a preview linking to a demo video), and removes it when that section is gone. Assisted-by: Claude (Opus 5.5 Medium) * ci: label video demos with gh instead of github-script The label workflow now runs the gh CLI in a shell step, like pr-draft-label.yml, so it needs no action outside the repository's allowed list. The video link check only counts videos uploaded for the PR itself, so a description linking other PRs' demos isn't labeled. Assisted-by: Claude (Opus 5.5 Medium) * ui: zoom demos in on details with a two-finger pinch demo.zoom pinches in on a control or a point, holds, and pinches back out to 1x, so a detail too small to read in the frame can be shown up close. It drives Chromium's own pinch gesture, so the page's fixed bars stay put, and works at desktop and phone sizes alike. The pinch is evened out where Chromium misses: one anchored near the top does not zoom at all, so it starts lower, and one near the bottom overshoots its ratio, so a small pinch from the middle corrects it. The way out reads the real scale until the page is back at 1x, which the next gesture needs. The overlay also draws a fingertip per touch, so a pinch on a touch device shows as two fingers. Assisted-by: Claude (Opus 5.5 Medium) * ui: wait out a cold dev server before a demo starts A fresh checkout's dev server optimizes its dependencies on the first load and reloads the page, which can take well past an action's timeout, so a demo failed before it started. The demo now waits up to 90 seconds for the app; the recording only starts once it is ready. Assisted-by: Claude (Opus 5.5 Medium) * ui: record a video per test in a PR demo, shown in a table A large PR can split its demo into several shorter videos: each test() in the ts demo block is recorded as its own video (-1.mp4, ...), and the Demo section lays their previews out in a table, each column headed by its test's title. GIF previews now keep their longer side at DEMO_PREVIEW_WIDTH and shrink until they are under 5 MB, since GitHub's image proxy refuses anything larger and shows a broken image instead. Importing traces works on a phone viewport too, where Settings sits in the sidebar under the page until the menu key slides it aside. Assisted-by: Claude (Opus 5.5 Medium) * ui: fix demo trace imports on desktop and lay landscape demos out two to a row Importing traces failed on a desktop whose collapsed sidebar sits under the page: it now expands the sidebar from the page's menu key on a phone or from the sidebar's logo on a desktop, whichever makes Settings reachable. Afterwards it forgets the expanded sidebar, so a demo starts with it collapsed, as on a first visit. A PR demo with several videos shows up to four portrait (phone) videos to a row, or two landscape ones, so each preview stays readable. Assisted-by: Claude (Opus 5.5 Medium) * ui: only record other people's PR demos at a reviewed head commit demo:pr runs the PR's code and demo script on this machine, with the gh and hf logins, so a PR by someone else now needs --trust naming the head commit that was read, and is refused again after new pushes. Recordings also launch Chromium with its sandbox on, which Playwright turns off by default. Assisted-by: Claude (Opus 5.5 Medium) --- .github/workflows/pr-video-demo-label.yml | 37 ++ tools/ui/.env.example | 18 + tools/ui/demo/.gitignore | 5 + tools/ui/demo/README.md | 133 ++++++ tools/ui/demo/cli.ts | 411 ++++++++++++++++++ tools/ui/demo/fixtures/demo-config.ts | 49 +++ tools/ui/demo/fixtures/mcp-servers.ts | 91 ++++ tools/ui/demo/fixtures/traces.ts | 312 ++++++++++++++ tools/ui/demo/helpers.ts | 500 ++++++++++++++++++++++ tools/ui/demo/playwright.config.ts | 87 ++++ tools/ui/demo/recorder.ts | 109 +++++ tools/ui/demo/storage.ts | 92 ++++ tools/ui/package.json | 5 + 13 files changed, 1849 insertions(+) create mode 100644 .github/workflows/pr-video-demo-label.yml create mode 100644 tools/ui/demo/.gitignore create mode 100644 tools/ui/demo/README.md create mode 100644 tools/ui/demo/cli.ts create mode 100644 tools/ui/demo/fixtures/demo-config.ts create mode 100644 tools/ui/demo/fixtures/mcp-servers.ts create mode 100644 tools/ui/demo/fixtures/traces.ts create mode 100644 tools/ui/demo/helpers.ts create mode 100644 tools/ui/demo/playwright.config.ts create mode 100644 tools/ui/demo/recorder.ts create mode 100644 tools/ui/demo/storage.ts diff --git a/.github/workflows/pr-video-demo-label.yml b/.github/workflows/pr-video-demo-label.yml new file mode 100644 index 000000000000..9b3165fa22ab --- /dev/null +++ b/.github/workflows/pr-video-demo-label.yml @@ -0,0 +1,37 @@ +name: "PR video demo label" + +on: + pull_request_target: + types: [opened, edited, reopened, synchronize] + +cache-mode: none +permissions: + pull-requests: write + issues: write + +jobs: + video-demo-label: + runs-on: ubuntu-slim + steps: + - name: Add or remove the video demo label + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_URL: ${{ github.event.pull_request.html_url }} + PR_NUMBER: ${{ github.event.pull_request.number }} + # passed through env, so the body is only matched, never run, and is safe to read from a fork + BODY: ${{ github.event.pull_request.body }} + HAS_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'video demo') }} + run: | + # sections written by demo:pr (see tools/ui/demo/cli.ts): the managed comment + # markers, or a preview linking to a demo video uploaded for this PR + has_demo=false + if { grep -qF '' <<< "$BODY" && grep -qF '' <<< "$BODY"; } || + grep -qE "/resolve/videos/pr/$PR_NUMBER/[0-9a-f]{10}(-[0-9]+)?\.mp4" <<< "$BODY"; then + has_demo=true + fi + + if [ "$has_demo" = true ] && [ "$HAS_LABEL" = false ]; then + gh pr edit "$PR_URL" --add-label 'video demo' + elif [ "$has_demo" = false ] && [ "$HAS_LABEL" = true ]; then + gh pr edit "$PR_URL" --remove-label 'video demo' + fi diff --git a/tools/ui/.env.example b/tools/ui/.env.example index a1e6c54915ba..a561a68a6560 100644 --- a/tools/ui/.env.example +++ b/tools/ui/.env.example @@ -1,3 +1,21 @@ VITE_PUBLIC_APP_NAME='llama-ui' VITE_DEBUG='true' VITE_PUBLIC_SERVER_ORIGIN='http://localhost:8033' + +# Demo videos (npm run demo:pr / demo:docs), see demo/README.md +# Where demo scripts and videos are stored; unset keeps videos in demo/videos +# DEMO_STORAGE='hf://buckets//' +# DEMO_STORAGE='file:///path/to/dir' +# Public base URL of DEMO_STORAGE, used to link videos in PR descriptions +# DEMO_STORAGE_PUBLIC_URL='https://...' +# HF dataset for demo conversations (llama.app traces): `demo:traces` pushes the +# built-in ones there, and demos can import any .jsonl from it +# DEMO_TRACES_DATASET='/' +# DEMO_GITHUB_REPO='ggml-org/llama.cpp' +# DEMO_SERVER_ORIGIN='http://localhost:9931' +# DEMO_LLAMA_SERVER='../../build/bin/llama-server' +# DEMO_COLOR_SCHEME='light' +# Device pixel ratio demos render and record at (1440x900 viewport) +# DEMO_SCALE='2' +# Longer side of the GIF preview, in pixels; it shrinks further to stay under 5 MB +# DEMO_PREVIEW_WIDTH='800' diff --git a/tools/ui/demo/.gitignore b/tools/ui/demo/.gitignore new file mode 100644 index 000000000000..5a02216c4546 --- /dev/null +++ b/tools/ui/demo/.gitignore @@ -0,0 +1,5 @@ +.runs/ +test-results/ +videos/ +*.demo.ts +/traces/ diff --git a/tools/ui/demo/README.md b/tools/ui/demo/README.md new file mode 100644 index 000000000000..70617780da67 --- /dev/null +++ b/tools/ui/demo/README.md @@ -0,0 +1,133 @@ +# Demo videos + +Scripted Playwright walkthroughs of the UI, exported as mp4 with an animated GIF preview, for pull requests and documentation. The harness lives here; the demo scripts don't: + +- **PR demos** live in the PR description, as a `ts demo` code block. +- **Docs demos** live in the storage target, under `scripts/docs/`. + +``` +demo/ +├── cli.ts demo:pr, demo:docs and demo:traces +├── storage.ts storage targets (hf://, file://) +├── helpers.ts `test` with a `demo` fixture: cursor, captions, human-like timing +├── recorder.ts screencast recording and mp4 export +├── fixtures/ fixture MCP servers, seeded settings and demo conversations +└── videos/ output (git-ignored) +``` + +## Setup + +### Tools + +| Tool | Used for | +| ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Node 22.18+ | runs `demo/cli.ts` directly (TypeScript type stripping) | +| `npx playwright install chromium` | the browser demos record in, with its sandbox on | +| `ffmpeg` | mp4 encoding and the GIF preview | +| `gh`, logged in | reads the PR description and writes the `## Demo` section, so you need edit rights on the PR | +| `hf`, logged in with a write token (`hf auth login`) | uploads to the bucket and the traces dataset | +| `llama-server` | `build/bin/llama-server`, or one already running at `DEMO_SERVER_ORIGIN` (default `:9931`). It runs in router mode, so demos can pick any model in the local cache; scripts that send messages need at least one. | + +`gh` is only needed for `demo:pr`, and `hf` only with the bucket or dataset below. Without them, the video stays in `demo/videos/`. + +Videos are recorded through Chromium's screencast at `DEMO_SCALE` (2x by default, so 2880x1800) and 60 fps, starting once the page has settled. + +### Hugging Face bucket (videos, previews, scripts) + +The bucket must be public, since GitHub loads the preview from it: + +```bash +hf buckets create /llama-ui-demos +``` + +Then set it in `tools/ui/.env` (see `.env.example`): + +```bash +DEMO_STORAGE='hf://buckets//llama-ui-demos' +DEMO_STORAGE_PUBLIC_URL='https://huggingface.co/buckets//llama-ui-demos/resolve' +``` + +Layout: `videos/pr//.{mp4,gif,demo.ts}`, `videos/docs/.{mp4,gif}` and `scripts/docs/*.demo.ts` (the docs scripts). Without `DEMO_STORAGE_PUBLIC_URL`, videos are uploaded but not linked in the PR. `DEMO_STORAGE='file:///some/dir'` stores them in a local folder instead. + +### Hugging Face dataset (conversations, optional) + +Only needed for demos that import conversations other than the built-in ones, or to publish the built-in ones with `npm run demo:traces`. It can be private: it is read with your `hf` login. + +```bash +hf repos create /llama-ui-demo-traces --repo-type dataset --private +``` + +```bash +DEMO_TRACES_DATASET='/llama-ui-demo-traces' +``` + +### Other storage + +Only Hugging Face buckets and `file://` folders are supported for now. S3, R2 or similar would be one more class in `storage.ts` (`upload` and `downloadDir`) plus the matching `DEMO_STORAGE_PUBLIC_URL`. + +## Recording a pull request + +Add the script to the PR description: + +````md +
+Demo script + +```ts demo +test('demo', async ({ demo, page }) => { + await demo.click(page.getByRole('button', { exact: true, name: 'Settings' })); + await demo.caption('What this shows'); +}); +``` + +
+```` + +Then run: + +```bash +npm run demo:pr -- 31303 +``` + +A PR by someone else runs its code and demo script on your machine, with your `gh` and `hf` logins, so `demo:pr` refuses it until you've read its diff and demo block and pass the head commit you read: `npm run demo:pr -- 31303 --trust `. If the PR gets new commits, it refuses again. + +It checks out the PR head in a temporary worktree, records the script against it, uploads `videos/pr//.{mp4,gif,demo.ts}` and adds or updates a `## Demo` section in the PR description with the preview linking to the video. + +A large PR can show its changes in several shorter videos: each `test()` in the block is recorded as its own video (`-1.mp4`, `-2.mp4`, ...), and the `## Demo` section shows their previews in a table, each headed by its test's title: up to four portrait (phone) videos to a row, or two landscape ones. Helpers and `test.use` at the top of the block are shared by all of them. + +````ts +test('Header and safe areas', async ({ demo, page }) => { + // ... +}); + +test('Sidebar gestures', async ({ demo, page }) => { + // ... +}); +``` GitHub has no API for attachments, so for an inline player drag `demo/videos/pr-.mp4` into the description instead. + +## Recording docs + +```bash +npm run demo:docs +```` + +Records every `scripts/docs/*.demo.ts` from the storage target and uploads `videos/docs/.{mp4,gif}`. + +## Writing a demo + +Scripts get `test` and `expect` from `helpers.ts` without importing them. `demo.click`, `demo.hover` and `demo.type` glide a visible cursor to the target before acting, with timing varied like a person's (seeded per demo, so re-takes match); `demo.scroll` turns the mouse wheel over a target; `demo.zoom` pinches in on a control or a point to show a detail too small to read in the frame, holds, and pinches back out (with the fingertips drawn on a touch device); `demo.caption` shows a caption at the top of the frame. Use `test.use({ settings, startPath })` to seed other settings or start on another route. + +## Conversations + +Demos that need existing chats import them with `test.use({ traces: ['long-chat'] })`, through Settings > Import/Export before the video starts. Traces use the export format (llama.app JSONL), so the Hub shows them as Llama traces: + +- `fixtures/traces.ts` generates the built-in ones: `long-chat` (36 tall exchanges) and `agentic-debugging` (long reasoning, shell output and code blocks). +- Any other name is downloaded as `.jsonl` from `DEMO_TRACES_DATASET`, e.g. an exported conversation uploaded there. + +`npm run demo:traces` writes the built-in traces to `demo/traces/` and pushes them to `DEMO_TRACES_DATASET`. + +## Trying a script locally + +To try a script locally, put it in `demo/` as `.demo.ts`, importing from `./helpers.ts`, and run `npm run demo:record -- demo/`. `npm run demo:codegen` opens Playwright codegen against `:5180` for a first draft. + +The config starts, or reuses when already running, the fixture MCP servers on `:8765`, `llama-server` on `:9931` (router mode, `--tools all`) and the UI's vite dev server. diff --git a/tools/ui/demo/cli.ts b/tools/ui/demo/cli.ts new file mode 100644 index 000000000000..a7835013b8c4 --- /dev/null +++ b/tools/ui/demo/cli.ts @@ -0,0 +1,411 @@ +/** + * Records demo videos for pull requests and docs. + * + * node demo/cli.ts pr record the ```ts demo block from a PR description, a video per test + * (--trust to record someone else's PR at that head commit) + * node demo/cli.ts docs record every docs/*.demo.ts script in DEMO_STORAGE + * node demo/cli.ts traces write the built-in traces, and push them to DEMO_TRACES_DATASET + * + * Settings come from tools/ui/.env (see .env.example). + */ + +import { DEMO_TRACES, generateTrace } from './fixtures/traces.ts'; +import { type DemoStorage, getDemoStorage, getPublicUrl } from './storage.ts'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync +} from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; + +const DEMO_DIR = import.meta.dirname; +const UI_DIR = resolve(DEMO_DIR, '..'); +const RUNS_DIR = join(DEMO_DIR, '.runs'); +const VIDEOS_DIR = join(DEMO_DIR, 'videos'); +const TRACES_DIR = join(DEMO_DIR, 'traces'); +const DEMO_BLOCK = /```(?:ts|typescript)[ \t]+demo[ \t]*\r?\n([\s\S]*?)```/; +const HELPERS_IMPORT = /^import\s[^;]*from\s+['"][^'"]*helpers(?:\.ts)?['"];?\s*$/gm; +const SECTION_START = ''; +const SECTION_END = ''; +/** GitHub's image proxy refuses larger images, so a bigger preview shows as a broken image. */ +const PREVIEW_MAX_BYTES = 5 * 1024 * 1024; +const envFile = join(UI_DIR, '.env'); + +if (existsSync(envFile)) process.loadEnvFile(envFile); + +const REPO = process.env.DEMO_GITHUB_REPO ?? 'ggml-org/llama.cpp'; + +function run( + cmd: string, + args: string[], + opts: { cwd?: string; env?: NodeJS.ProcessEnv } = {} +): string { + return execFileSync(cmd, args, { + cwd: opts.cwd ?? UI_DIR, + encoding: 'utf8', + env: opts.env + }).trim(); +} + +/** Writes a demo script into .runs/, importing the harness helpers. */ +function writeDemoScript(name: string, source: string): string { + const file = join(RUNS_DIR, `${name}.demo.ts`); + const helpers = relative(RUNS_DIR, join(DEMO_DIR, 'helpers.ts')); + const body = source.replace(HELPERS_IMPORT, '').trim(); + + mkdirSync(RUNS_DIR, { recursive: true }); + writeFileSync(file, `import { expect, test } from './${helpers}';\n\n${body}\n`); + + return file; +} + +async function freePort(): Promise { + return new Promise((done, fail) => { + const server = createServer(); + + server.once('error', fail); + server.listen(0, () => { + const { port } = server.address() as { port: number }; + + server.close(() => done(port)); + }); + }); +} + +/** A video recorded from one `test()` of a script. */ +interface Video { + mp4: string; + title: string; +} + +/** Records the given scripts; returns a video for each of their tests, in order. */ +async function record(scripts: string[], uiDir = UI_DIR): Promise { + const port = await freePort(); + const manifests = scripts.map((s) => join(VIDEOS_DIR, `${basename(s, '.demo.ts')}.videos.jsonl`)); + + for (const manifest of manifests) rmSync(manifest, { force: true }); + + execFileSync( + 'npx', + [ + 'playwright', + 'test', + '--config=demo/playwright.config.ts', + ...scripts.map((s) => relative(UI_DIR, s)) + ], + { + cwd: UI_DIR, + env: { ...process.env, DEMO_UI_DIR: uiDir, DEMO_UI_PORT: String(port) }, + stdio: 'inherit' + } + ); + + // helpers.ts lists each video it encodes in the script's manifest + return manifests.flatMap((manifest) => + readFileSync(manifest, 'utf8') + .trim() + .split('\n') + .map((line) => JSON.parse(line) as Video) + ); +} + +/** + * Animated GIF preview that GitHub renders inline, its longer side DEMO_PREVIEW_WIDTH (so a + * phone's portrait video is that tall), shrunk until it fits under PREVIEW_MAX_BYTES. + */ +function makePreview(mp4: string): string { + const gif = mp4.replace(/\.mp4$/, '.gif'); + + for (let side = Number(process.env.DEMO_PREVIEW_WIDTH ?? 800); ; side = Math.round(side * 0.8)) { + execFileSync('ffmpeg', [ + '-y', + '-loglevel', + 'error', + '-i', + mp4, + '-vf', + `fps=10,scale='if(gt(iw,ih),${side},-2)':'if(gt(iw,ih),-2,${side})':flags=lanczos,split[a][b];[a]palettegen=stats_mode=diff[p];[b][p]paletteuse=dither=bayer:bayer_scale=5:diff_mode=rectangle`, + '-loop', + '0', + gif + ]); + + if (statSync(gif).size <= PREVIEW_MAX_BYTES || side < 320) return gif; + } +} + +function upload(storage: DemoStorage, files: Record): void { + for (const [key, file] of Object.entries(files)) { + storage.upload(file, key); + console.log(`uploaded ${key}`); + } +} + +/** + * Public URL with a content hash, so a re-recorded file at the same key gets a new + * URL: GitHub's image proxy caches previews by URL. + */ +function versionedUrl(key: string, file: string): null | string { + const url = getPublicUrl(key); + const version = createHash('sha1').update(readFileSync(file)).digest('hex').slice(0, 10); + + return url && `${url}?v=${version}`; +} + +/** Whether a video is taller than wide, like a phone's. */ +function isPortrait(mp4: string): boolean { + const [width, height] = run('ffprobe', [ + '-v', + 'error', + '-select_streams', + 'v:0', + '-show_entries', + 'stream=width,height', + '-of', + 'csv=p=0', + mp4 + ]) + .split(',') + .map(Number); + + return height > width; +} + +/** + * One preview, or a table of them when the script has several tests, each headed by its + * test's title: up to four portrait videos to a row, or two landscape ones, so each stays + * large enough to read. + */ +function demoSection( + links: { gif: string; mp4: string; title: string }[], + sha: string, + columns: number +): string { + const preview = ({ gif, mp4, title }: (typeof links)[number]) => + `[![${title.replace(/[[\]|]/g, '')}](${gif})](${mp4})`; + const cell = (text: string) => text.replace(/\|/g, '\\|'); + const rows = Array.from({ length: Math.ceil(links.length / columns) }, (_, i) => + links.slice(i * columns, (i + 1) * columns) + ); + const pad = (cells: string[]) => [...cells, ...Array(columns - cells.length).fill(' ')]; + const body = + links.length === 1 + ? [preview({ ...links[0], title: 'Demo' })] + : rows.flatMap((row, i) => [ + // a table has one header row, so later rows title their videos in bold + `| ${pad(row.map((l) => (i === 0 ? cell(l.title) : `**${cell(l.title)}**`))).join(' | ')} |`, + ...(i === 0 ? [`|${Array(columns).fill('---').join('|')}|`] : []), + `| ${pad(row.map(preview)).join(' | ')} |` + ]); + const note = + links.length === 1 ? 'the preview for the full video' : 'a preview for its full video'; + + return [ + SECTION_START, + ...body, + '', + `Recorded at ${sha.slice(0, 10)}; click ${note}.`, + SECTION_END + ].join('\n'); +} + +/** Replaces the managed demo section, or adds a `## Demo` section before the details. */ +function withDemoSection(body: string, section: string): string { + const start = body.indexOf(SECTION_START); + const end = body.indexOf(SECTION_END); + + if (start !== -1 && end > start) { + return body.slice(0, start) + section + body.slice(end + SECTION_END.length); + } + + const block = `## Demo\n\n${section}\n\n`; + const anchor = body.search(/^## (Additional information|Requirements)\b/m); + + return anchor === -1 + ? `${body.trimEnd()}\n\n${block}` + : body.slice(0, anchor) + block + body.slice(anchor); +} + +async function recordPr(number: string, trustedSha?: string): Promise { + const pr = JSON.parse( + run('gh', ['pr', 'view', number, '--repo', REPO, '--json', 'author,body,headRefOid']) + ) as { author: { login: string }; body: string; headRefOid: string }; + const source = DEMO_BLOCK.exec(pr.body)?.[1]; + + if (!source) throw new Error(`PR #${number} has no \`\`\`ts demo block in its description`); + + const sha = pr.headRefOid; + + // the PR's code and demo script run on this machine, with the gh and hf logins, + // so someone else's PR only runs at a head commit you've read and passed to --trust + if (pr.author.login !== run('gh', ['api', 'user', '--jq', '.login'])) { + if (!trustedSha) { + throw new Error( + `PR #${number} is by ${pr.author.login}, and recording it runs its code here: ` + + `read its diff and demo block, then run again with --trust ${sha.slice(0, 10)}` + ); + } + + if (trustedSha.length < 7 || !sha.startsWith(trustedSha)) { + throw new Error(`PR #${number} head is ${sha.slice(0, 10)}, not the trusted ${trustedSha}`); + } + } + + const worktree = join(tmpdir(), `llama-ui-demo-pr-${number}`); + + run('git', ['fetch', '--no-tags', `https://github.com/${REPO}.git`, `pull/${number}/head`]); + + if (run('git', ['rev-parse', 'FETCH_HEAD']) !== sha) { + throw new Error(`PR #${number} head moved while fetching, run again`); + } + + rmSync(worktree, { force: true, recursive: true }); + run('git', ['worktree', 'prune']); + run('git', ['worktree', 'add', '--detach', worktree, sha]); + symlinkSync(join(UI_DIR, 'node_modules'), join(worktree, 'tools/ui/node_modules')); + + const name = `pr-${number}`; + const script = writeDemoScript(name, source); + + try { + const videos = (await record([script], join(worktree, 'tools/ui'))).map((video) => ({ + ...video, + gif: makePreview(video.mp4) + })); + const storage = getDemoStorage(); + + for (const { gif, mp4 } of videos) console.log(`\nvideo: ${mp4}\npreview: ${gif}`); + + if (!storage) { + console.log('DEMO_STORAGE is not set: drag the mp4 into the PR description.'); + + return; + } + + const prefix = `videos/pr/${number}/${sha.slice(0, 10)}`; + // a script with several tests gets -1.mp4, -2.mp4 and so on + const keys = videos.map((_, i) => (videos.length === 1 ? prefix : `${prefix}-${i + 1}`)); + + upload(storage, { [`${prefix}.demo.ts`]: script }); + videos.forEach(({ gif, mp4 }, i) => { + upload(storage, { [`${keys[i]}.gif`]: gif, [`${keys[i]}.mp4`]: mp4 }); + }); + + if (!getPublicUrl(prefix)) { + console.log('DEMO_STORAGE_PUBLIC_URL is not set: not linking the video in the PR.'); + + return; + } + + const links = videos.map(({ gif, mp4, title }, i) => ({ + gif: versionedUrl(`${keys[i]}.gif`, gif) ?? '', + mp4: versionedUrl(`${keys[i]}.mp4`, mp4) ?? '', + title + })); + + execFileSync('gh', ['pr', 'edit', number, '--repo', REPO, '--body-file', '-'], { + cwd: UI_DIR, + input: withDemoSection( + pr.body, + demoSection(links, sha, Math.min(links.length, isPortrait(videos[0].mp4) ? 4 : 2)) + ), + stdio: ['pipe', 'inherit', 'inherit'] + }); + console.log(`linked the demo in https://github.com/${REPO}/pull/${number}`); + } finally { + rmSync(script, { force: true }); + run('git', ['worktree', 'remove', '--force', worktree]); + } +} + +async function recordDocs(): Promise { + const storage = getDemoStorage(); + + if (!storage) throw new Error('DEMO_STORAGE is not set: docs demo scripts are read from it'); + + const downloaded = join(RUNS_DIR, 'docs-source'); + + rmSync(downloaded, { force: true, recursive: true }); + + if (!storage.downloadDir('scripts/docs', downloaded)) { + throw new Error('no scripts/docs/*.demo.ts in DEMO_STORAGE'); + } + + const scripts = readdirSync(downloaded) + .filter((f) => f.endsWith('.demo.ts')) + .map((f) => + writeDemoScript(basename(f, '.demo.ts'), readFileSync(join(downloaded, f), 'utf8')) + ); + + try { + const videos = await record(scripts); + + for (const { mp4 } of videos) { + const name = basename(mp4, '.mp4'); + + upload(storage, { + [`videos/docs/${name}.gif`]: makePreview(mp4), + [`videos/docs/${name}.mp4`]: mp4 + }); + } + } finally { + rmSync(RUNS_DIR, { force: true, recursive: true }); + } +} + +/** Writes the built-in traces to demo/traces/ and uploads them to the traces dataset. */ +function writeTraces(): void { + rmSync(TRACES_DIR, { force: true, recursive: true }); + mkdirSync(TRACES_DIR, { recursive: true }); + + for (const name of Object.keys(DEMO_TRACES)) { + writeFileSync(join(TRACES_DIR, `${name}.jsonl`), `${generateTrace(name)}\n`); + console.log(`wrote demo/traces/${name}.jsonl`); + } + + const dataset = process.env.DEMO_TRACES_DATASET; + + if (!dataset) { + console.log('DEMO_TRACES_DATASET is not set: not uploading.'); + + return; + } + + execFileSync( + 'hf', + [ + 'upload', + dataset, + TRACES_DIR, + '.', + '--repo-type', + 'dataset', + '--commit-message', + 'Update demo traces' + ], + { stdio: 'inherit' } + ); +} + +const [command, arg, ...flags] = process.argv.slice(2); +const trustIndex = flags.indexOf('--trust'); + +if (command === 'pr' && arg) { + await recordPr(arg, trustIndex === -1 ? undefined : flags[trustIndex + 1]); +} else if (command === 'docs') { + await recordDocs(); +} else if (command === 'traces') { + writeTraces(); +} else { + console.error('usage: node demo/cli.ts pr [--trust ] | docs | traces'); + process.exit(1); +} diff --git a/tools/ui/demo/fixtures/demo-config.ts b/tools/ui/demo/fixtures/demo-config.ts new file mode 100644 index 000000000000..d4a272f6cdd4 --- /dev/null +++ b/tools/ui/demo/fixtures/demo-config.ts @@ -0,0 +1,49 @@ +/** Shared fixture data for demo recordings (used by both Node and the browser seed). */ + +export const DEMO_MCP_PORT = 8765; + +export interface DemoMcpServer { + id: string; + name: string; + title: string; + tools: { name: string; description: string; result: string }[]; +} + +export const DEMO_MCP_SERVERS: DemoMcpServer[] = [ + { + id: 'weather', + name: 'weather', + title: 'Weather', + tools: [ + { + description: 'Get the weather forecast for a city', + name: 'get_forecast', + result: 'Sunny, 22°C' + }, + { + description: 'List active weather alerts for a region', + name: 'get_alerts', + result: 'No active alerts' + } + ] + }, + { + id: 'notes', + name: 'notes', + title: 'Notes', + tools: [ + { description: 'Search notes by keyword', name: 'search_notes', result: 'No notes found' }, + { description: 'Create a new note', name: 'create_note', result: 'Note created' }, + { description: 'List recent notes', name: 'list_notes', result: 'No notes yet' } + ] + } +]; + +/** `mcpServers` setting value pointing the UI at the fixture servers. */ +export function demoMcpServerSettings(): object[] { + return DEMO_MCP_SERVERS.map((server) => ({ + enabled: true, + id: `demo-${server.id}`, + url: `http://localhost:${DEMO_MCP_PORT}/${server.id}/mcp` + })); +} diff --git a/tools/ui/demo/fixtures/mcp-servers.ts b/tools/ui/demo/fixtures/mcp-servers.ts new file mode 100644 index 000000000000..15529e4a314e --- /dev/null +++ b/tools/ui/demo/fixtures/mcp-servers.ts @@ -0,0 +1,91 @@ +/** + * Fixture MCP servers for demo recordings. + * + * Serves a few stateless Streamable HTTP MCP servers with canned tools, so + * demos show MCP groups without depending on real third-party servers. + * + * Usage: node demo/fixtures/mcp-servers.ts + */ + +import { DEMO_MCP_PORT, DEMO_MCP_SERVERS, type DemoMcpServer } from './demo-config.ts'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import { z } from 'zod'; + +function buildServer(fixture: DemoMcpServer): McpServer { + const server = new McpServer({ name: fixture.name, title: fixture.title, version: '1.0.0' }); + + for (const tool of fixture.tools) { + server.registerTool( + tool.name, + { description: tool.description, inputSchema: { query: z.string().optional() } }, + async () => ({ content: [{ text: tool.result, type: 'text' }] }) + ); + } + + return server; +} + +function setCorsHeaders(res: ServerResponse): void { + res.setHeader('Access-Control-Allow-Origin', '*'); + res.setHeader('Access-Control-Allow-Headers', '*'); + res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS'); + res.setHeader('Access-Control-Expose-Headers', 'mcp-session-id, mcp-protocol-version'); + // the UI runs with COEP require-corp + res.setHeader('Cross-Origin-Resource-Policy', 'cross-origin'); +} + +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + + for await (const chunk of req) chunks.push(chunk as Buffer); + + const raw = Buffer.concat(chunks).toString(); + + return raw ? JSON.parse(raw) : undefined; +} + +const httpServer = createServer(async (req, res) => { + setCorsHeaders(res); + + if (req.method === 'OPTIONS') { + res.writeHead(204).end(); + + return; + } + + const path = new URL(req.url ?? '/', 'http://localhost').pathname; + const fixture = DEMO_MCP_SERVERS.find((s) => path === `/${s.id}/mcp`); + + if (!fixture) { + res.writeHead(path === '/health' ? 200 : 404).end(); + + return; + } + + if (req.method !== 'POST') { + // stateless: no standalone SSE stream or session teardown + res.writeHead(405).end(); + + return; + } + + const server = buildServer(fixture); + const transport = new StreamableHTTPServerTransport({ + enableJsonResponse: true, + sessionIdGenerator: undefined + }); + + res.on('close', () => { + transport.close(); + server.close(); + }); + + await server.connect(transport); + await transport.handleRequest(req, res, await readBody(req)); +}); + +httpServer.listen(DEMO_MCP_PORT, () => { + console.log(`demo MCP servers listening on http://localhost:${DEMO_MCP_PORT}`); +}); diff --git a/tools/ui/demo/fixtures/traces.ts b/tools/ui/demo/fixtures/traces.ts new file mode 100644 index 000000000000..cc7ed8979a95 --- /dev/null +++ b/tools/ui/demo/fixtures/traces.ts @@ -0,0 +1,312 @@ +/** + * Synthetic conversations for demos, in the UI's export format (llama.app JSONL + * traces): a `session` record followed by one `message` record per message. + * Deterministic, so a trace and the demos recorded from it don't drift. Messages + * carry no model, so the UI doesn't flag one the server doesn't have. + */ + +import { createHash } from 'node:crypto'; + +const HARNESS = 'llama.app'; +const START = Date.UTC(2026, 9, 1, 9, 0, 0); + +interface ToolCall { + function: { arguments: string; name: string }; + id: string; + type: 'function'; +} + +interface TraceMessage { + children: string[]; + content: string; + convId: string; + id: string; + parent: null | string; + reasoningContent?: string; + role: 'assistant' | 'system' | 'tool' | 'user'; + timestamp: number; + toolCallId?: string; + toolCalls?: ToolCall[]; + type: 'root' | 'text'; +} + +/** Builds one linear conversation, message by message. */ +class Trace { + private calls = 0; + private readonly id: string; + private readonly messages: TraceMessage[] = []; + private readonly name: string; + + assistant(content: string, reasoning?: string): this { + return this.push({ content, reasoningContent: reasoning, role: 'assistant' }); + } + + constructor(name: string) { + this.name = name; + this.id = uuid(name, 'conv'); + this.push({ content: '', role: 'system', type: 'root' }); + } + + toJsonl(): string { + const last = this.messages.at(-1)!; + const session = { + currNode: last.id, + harness: HARNESS, + id: this.id, + lastModified: last.timestamp, + name: this.name, + type: 'session' + }; + + return [session, ...this.messages.map((message) => ({ message, type: 'message' }))] + .map((record) => JSON.stringify(record)) + .join('\n'); + } + + /** An assistant turn calling one tool, followed by the tool's result. */ + tool(name: string, args: Record, result: string, reasoning?: string): this { + const id = uuid(this.name, `call-${this.calls++}`).replaceAll('-', ''); + + this.push({ + content: '', + reasoningContent: reasoning, + role: 'assistant', + toolCalls: [{ function: { arguments: JSON.stringify(args), name }, id, type: 'function' }] + }); + + return this.push({ content: result, role: 'tool', toolCallId: id }); + } + + user(content: string): this { + return this.push({ content, role: 'user' }); + } + + private push(message: Partial & Pick): this { + const parent = this.messages.at(-1); + const index = this.messages.length; + const entry: TraceMessage = { + children: [], + convId: this.id, + id: uuid(this.name, `msg-${index}`), + parent: parent?.id ?? null, + timestamp: START + index * 20_000, + type: 'text', + ...message + }; + + parent?.children.push(entry.id); + this.messages.push(entry); + + return this; + } +} + +function uuid(...parts: string[]): string { + const hex = createHash('sha1').update(parts.join(':')).digest('hex'); + + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-4${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +} + +const STEPS = [ + ['set up the project', 'cargo new forecast && cd forecast', 'Cargo.toml', 'toml'], + ['parse command line arguments', 'cargo add clap --features derive', 'src/cli.rs', 'rust'], + ['call the weather API', 'cargo add reqwest --features json,blocking', 'src/api.rs', 'rust'], + ['deserialize the response', 'cargo add serde --features derive', 'src/model.rs', 'rust'], + ['cache responses on disk', 'cargo add directories', 'src/cache.rs', 'rust'], + ['print a table', 'cargo add comfy-table', 'src/table.rs', 'rust'], + ['handle errors', 'cargo add anyhow', 'src/main.rs', 'rust'], + ['add colors', 'cargo add owo-colors', 'src/style.rs', 'rust'], + ['read a config file', 'cargo add toml', 'src/config.rs', 'rust'], + ['write tests', 'cargo add --dev assert_cmd predicates', 'tests/cli.rs', 'rust'], + ['set up CI', 'mkdir -p .github/workflows', '.github/workflows/ci.yml', 'yaml'], + ['publish a release', 'cargo install cargo-dist', 'dist-workspace.toml', 'toml'] +] as const; + +function codeFor(step: number, file: string, lang: string): string { + const lines = + lang === 'rust' + ? [ + `// ${file}`, + 'use anyhow::{Context, Result};', + '', + `pub fn step_${step}(input: &str) -> Result> {`, + ' let mut out = Vec::new();', + ' for (i, line) in input.lines().enumerate() {', + ' let trimmed = line.trim();', + ' if trimmed.is_empty() {', + ' continue;', + ' }', + ' out.push(format!("{i:>3}: {trimmed}"));', + ' }', + ` Ok(out).context("step ${step} failed")`, + '}' + ] + : lang === 'yaml' + ? [ + `# ${file}`, + 'on: [push, pull_request]', + 'jobs:', + ' test:', + ' runs-on: ubuntu-latest' + ] + : [`# ${file}`, `name = "forecast"`, `version = "0.${step}.0"`, 'edition = "2024"']; + + return `\`\`\`${lang}\n${lines.join('\n')}\n\`\`\``; +} + +/** A long chat with tall messages, for scrolling and landing demos. */ +function longChat(): Trace { + const trace = new Trace('Building a weather CLI in Rust'); + + for (let i = 0; i < 36; i++) { + const [goal, command, file, lang] = STEPS[i % STEPS.length]; + const round = Math.floor(i / STEPS.length); + const ask = + round === 0 ? `How do I ${goal}?` : `Can we revisit how we ${goal}? (pass ${round + 1})`; + + trace.user(ask); + trace.assistant( + [ + `To ${goal}, start from a clean working tree so the change stays easy to review.`, + '', + '```sh', + command, + '```', + '', + `Then add \`${file}\`:`, + '', + codeFor(i + 1, file, lang), + '', + 'A few things to keep in mind:', + '', + `- Keep \`${file}\` focused on one job, so the next step only touches what it needs.`, + '- Run `cargo clippy` after each step; it catches most mistakes early.', + '- Commit once it builds, so you can always go back to a working version.', + '', + i % 3 === 0 + ? `Next we can look at how to ${STEPS[(i + 1) % STEPS.length][0]}.` + : 'Let me know if anything here does not build.' + ].join('\n'), + `The user wants to ${goal}. Give the command, the file and a short checklist.` + ); + } + + return trace; +} + +function testLog(failing: boolean): string { + const lines = ['running 48 tests']; + + for (let i = 1; i <= 48; i++) { + const ok = !failing || (i !== 17 && i !== 31); + + lines.push( + `test cache::tests::entry_${String(i).padStart(2, '0')} ... ${ok ? 'ok' : 'FAILED'}` + ); + } + + if (failing) { + lines.push( + '', + 'failures:', + '', + '---- cache::tests::entry_17 stdout ----', + "thread 'cache::tests::entry_17' panicked at src/cache.rs:88:9:", + 'assertion `left == right` failed: entry expired too early', + ' left: Some(1759312800)', + ' right: Some(1759316400)', + '', + '---- cache::tests::entry_31 stdout ----', + "thread 'cache::tests::entry_31' panicked at src/cache.rs:88:9:", + 'assertion `left == right` failed: entry expired too early', + '', + 'test result: FAILED. 46 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out' + ); + } else { + lines.push('', 'test result: ok. 48 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out'); + } + + return `${lines.join('\n')}\n\n[exit code: ${failing ? 101 : 0}]`; +} + +function reasoning(topic: string, paragraphs: number): string { + const out: string[] = []; + + for (let i = 0; i < paragraphs; i++) { + out.push( + `${i === 0 ? `Looking at ${topic}.` : 'Another angle:'} The cache stores an expiry timestamp per entry, ` + + 'and the failing tests are the ones created right at an hour boundary. If the expiry is computed ' + + 'from local time while the check uses UTC, entries created near the boundary would expire an hour ' + + `early. Step ${i + 1}: confirm by reading how \`expires_at\` is built, then compare with \`is_fresh\`.` + ); + } + + return out.join('\n\n'); +} + +/** An agentic session with long reasoning, shell output and code, for nested scroller demos. */ +function agenticDebugging(): Trace { + const trace = new Trace('Fix the flaky cache tests'); + const fixed = Array.from({ length: 60 }, (_, i) => + i === 0 + ? 'use chrono::{DateTime, Duration, Utc};' + : i === 30 + ? ' let expires_at = Utc::now() + self.ttl; // was Local::now()' + : ` // cache.rs line ${i + 1}: entry handling stays the same` + ).join('\n'); + + trace.user('The cache tests fail on CI every few runs. Can you find out why and fix it?'); + + for (let round = 0; round < 3; round++) { + trace.tool( + 'exec_shell_command', + { command: 'cargo test cache -- --test-threads=1' }, + testLog(true), + reasoning('the failing cache tests', 4 + round) + ); + trace.tool( + 'exec_shell_command', + { command: "grep -n 'expires_at\\|is_fresh' src/cache.rs" }, + Array.from( + { length: 30 }, + (_, i) => `${80 + i}: let expires_at = Local::now() + self.ttl; // entry ${i}` + ).join('\n') + '\n\n[exit code: 0]', + reasoning('where the expiry is computed', 3) + ); + trace.assistant( + [ + `Round ${round + 1}: the expiry uses local time while the freshness check uses UTC.`, + '', + '```rust', + fixed, + '```', + '', + 'I switched the expiry to `Utc::now()` so both sides use the same clock.' + ].join('\n'), + reasoning('the fix', 5) + ); + trace.user(round < 2 ? 'Still flaky on CI, can you check again?' : 'Run the tests once more.'); + } + + trace.tool( + 'exec_shell_command', + { command: 'cargo test cache' }, + testLog(false), + reasoning('the final test run', 2) + ); + trace.assistant( + 'All 48 cache tests pass now, including the two that failed at the hour boundary.' + ); + + return trace; +} + +/** Built-in traces, by name. */ +export const DEMO_TRACES: Record Trace> = { + 'agentic-debugging': agenticDebugging, + 'long-chat': longChat +}; + +export function generateTrace(name: string): null | string { + return DEMO_TRACES[name]?.().toJsonl() ?? null; +} diff --git a/tools/ui/demo/helpers.ts b/tools/ui/demo/helpers.ts new file mode 100644 index 000000000000..c82b27069ec1 --- /dev/null +++ b/tools/ui/demo/helpers.ts @@ -0,0 +1,500 @@ +import { demoMcpServerSettings } from './fixtures/demo-config.ts'; +import { generateTrace } from './fixtures/traces.ts'; +import { ScreencastRecorder } from './recorder.ts'; +import { type CDPSession, type Locator, type Page, test as base } from '@playwright/test'; +import { execFileSync } from 'node:child_process'; +import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; + +const VIDEOS_DIR = resolve(import.meta.dirname, 'videos'); +/** How long a demo waits for the app to be ready before it starts recording. */ +const APP_READY_TIMEOUT_MS = 90000; +/** How far below the viewport's top a pinch has to start for Chromium to zoom at all. */ +const ZOOM_TOP_INSET_PX = 64; + +/** + * Draws a fake cursor and click ripples, since recorded videos don't include + * the OS pointer, and a fingertip per touch, so a pinch shows as two fingers. + * Runs in the page via addInitScript. + */ +function installCursorOverlay(): void { + const mount = () => { + const style = document.createElement('style'); + + style.textContent = ` + #demo-cursor { position: fixed; top: 0; left: 0; z-index: 2147483647; pointer-events: none; + width: 22px; height: 22px; margin: -3px 0 0 -3px; transition: transform 80ms ease-out; } + #demo-cursor.down { transform: scale(0.85); } + .demo-ripple { position: fixed; z-index: 2147483646; pointer-events: none; width: 36px; height: 36px; + margin: -18px 0 0 -18px; border-radius: 50%; background: rgba(59, 130, 246, 0.35); + animation: demo-ripple 450ms ease-out forwards; } + @keyframes demo-ripple { from { transform: scale(0.3); opacity: 1; } to { transform: scale(1.4); opacity: 0; } } + #demo-caption { position: fixed; left: 50%; top: 72px; z-index: 2147483647; pointer-events: none; + transform: translate(-50%, -8px); opacity: 0; transition: opacity 200ms ease, transform 200ms ease; + max-width: 80%; padding: 10px 18px; border-radius: 10px; background: rgba(17, 17, 17, 0.88); + color: #fff; font: 500 18px/1.4 system-ui, sans-serif; text-align: center; } + #demo-caption.visible { opacity: 1; transform: translate(-50%, 0); } + .demo-finger { position: fixed; z-index: 2147483646; pointer-events: none; width: 40px; height: 40px; + margin: -20px 0 0 -20px; border-radius: 50%; background: rgba(59, 130, 246, 0.3); + border: 2px solid rgba(59, 130, 246, 0.55); transition: opacity 200ms ease; } + `; + document.head.appendChild(style); + + const cursor = document.createElement('div'); + + cursor.id = 'demo-cursor'; + cursor.innerHTML = ``; + document.documentElement.appendChild(cursor); + + const caption = document.createElement('div'); + + caption.id = 'demo-caption'; + document.documentElement.appendChild(caption); + + document.addEventListener( + 'mousemove', + (e) => (cursor.style.translate = `${e.clientX}px ${e.clientY}px`), + true + ); + document.addEventListener( + 'mousedown', + (e) => { + cursor.classList.add('down'); + const ripple = document.createElement('div'); + + ripple.className = 'demo-ripple'; + ripple.style.left = `${e.clientX}px`; + ripple.style.top = `${e.clientY}px`; + document.documentElement.appendChild(ripple); + setTimeout(() => ripple.remove(), 500); + }, + true + ); + document.addEventListener('mouseup', () => cursor.classList.remove('down'), true); + + const fingers: HTMLDivElement[] = []; + const placeFingers = (e: TouchEvent) => { + Array.from(e.touches).forEach((touch, i) => { + const finger = (fingers[i] ??= Object.assign(document.createElement('div'), { + className: 'demo-finger' + })); + + finger.style.left = `${touch.clientX}px`; + finger.style.top = `${touch.clientY}px`; + finger.style.opacity = '1'; + document.documentElement.appendChild(finger); + }); + fingers.slice(e.touches.length).forEach((finger) => (finger.style.opacity = '0')); + }; + + for (const type of ['touchstart', 'touchmove', 'touchend', 'touchcancel']) { + document.addEventListener(type, placeFingers as EventListener, { + capture: true, + passive: true + }); + } + }; + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', mount); + } else { + mount(); + } +} + +/** Seeds UI settings once per demo, so later reloads keep what the demo changed. */ +function seedSettings(config: Record): void { + const key = 'LlamaUi.config'; + + if (localStorage.getItem(key) === null) { + localStorage.setItem(key, JSON.stringify(config)); + } +} + +/** Seeded randomness, so a demo moves the same way on every recording. */ +function createRandom(seed: string): () => number { + let state = [...seed].reduce((h, c) => Math.imul(h ^ c.charCodeAt(0), 2654435761), 1); + + return () => { + state = (state + 0x6d2b79f5) | 0; + + let t = Math.imul(state ^ (state >>> 15), 1 | state); + + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +const easeInOut = (t: number) => (t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2); + +/** + * Drives the page like a person: the cursor glides along a slight arc with + * Fitts-like durations, settles briefly before pressing, and typing and pauses + * vary a little. + */ +export class Demo { + private cdp: CDPSession | undefined; + private readonly page: Page; + private readonly random: () => number; + private x = 0; + private y = 0; + + /** Shows a caption at the top of the frame for `ms`, clear of the chat form. */ + async caption(text: string, ms = 2200): Promise { + await this.page.evaluate((t) => { + const el = document.getElementById('demo-caption'); + + if (!el) return; + + el.textContent = t; + el.classList.add('visible'); + }, text); + await this.page.waitForTimeout(ms); + await this.page.evaluate(() => + document.getElementById('demo-caption')?.classList.remove('visible') + ); + await this.page.waitForTimeout(250); + } + + async click(target: Locator, ms = 400): Promise { + await this.moveTo(target); + await this.page.waitForTimeout(this.between(80, 160)); + await this.page.mouse.down(); + await this.page.waitForTimeout(this.between(50, 90)); + await this.page.mouse.up(); + await this.pause(ms); + } + + constructor(page: Page, seed: string) { + this.page = page; + this.random = createRandom(seed); + + const viewport = page.viewportSize(); + + this.x = (viewport?.width ?? 0) / 2; + this.y = (viewport?.height ?? 0) / 2; + } + + async hover(target: Locator, ms = 350): Promise { + await this.moveTo(target); + await this.pause(ms); + } + + /** Glides the cursor to a point near the center of `target`. */ + async moveTo(target: Locator): Promise { + await target.scrollIntoViewIfNeeded(); + const box = await target.boundingBox(); + + if (!box) throw new Error(`demo: ${target} has no bounding box`); + + const x = box.x + box.width * this.between(0.4, 0.6); + const y = box.y + box.height * this.between(0.4, 0.6); + const dx = x - this.x; + const dy = y - this.y; + const distance = Math.hypot(dx, dy); + const duration = 110 + 55 * Math.log2(distance / 40 + 1); + // bow the path a little to one side, like a wrist turning + const bow = distance * this.between(-0.08, 0.08); + const fromX = this.x; + const fromY = this.y; + const started = Date.now(); + + for (;;) { + const t = Math.min((Date.now() - started) / duration, 1); + const p = easeInOut(t); + const arc = Math.sin(Math.PI * p) * bow; + + this.x = fromX + dx * p - (dy / (distance || 1)) * arc; + this.y = fromY + dy * p + (dx / (distance || 1)) * arc; + await this.page.mouse.move(this.x, this.y); + + if (t === 1) break; + + await this.page.waitForTimeout(8); + } + } + + /** Waits about `ms`, varied by up to 15% so pauses don't feel metronomic. */ + async pause(ms = 800): Promise { + await this.page.waitForTimeout(ms * this.between(0.85, 1.15)); + } + + async press(key: string, ms = 600): Promise { + await this.page.keyboard.press(key); + await this.pause(ms); + } + + /** Scrolls the wheel over `target` by `dy` pixels (negative scrolls up), in small steps like a trackpad. */ + async scroll(target: Locator, dy: number, ms = 500): Promise { + await this.moveTo(target); + + const step = Math.sign(dy) * 40; + + for (let done = 0; Math.abs(done) < Math.abs(dy); done += step) { + await this.page.mouse.wheel(0, step); + await this.page.waitForTimeout(16); + } + + await this.pause(ms); + } + + /** Types quickly with uneven key timing, a little slower after spaces and punctuation. */ + async type(target: Locator, text: string, ms = 500): Promise { + await this.click(target, 200); + + for (const char of text) { + await this.page.keyboard.type(char); + await this.page.waitForTimeout( + this.between(15, 45) + (/[\s.,!?]/.test(char) ? this.between(20, 60) : 0) + ); + } + + await this.pause(ms); + } + + /** + * Pinches in on `target`, a control or a point in the viewport, holds about `ms`, and pinches + * back out to 1x: a two-finger zoom onto a detail too small to read in the frame. On a touch + * device the fingertips show. + */ + async zoom(target: Locator | { x: number; y: number }, scale = 2.5, ms = 1400): Promise { + const point = 'x' in target ? target : await this.center(target); + // a pinch anchored in the viewport's top band does not zoom at all + const at = { x: point.x, y: Math.max(point.y, ZOOM_TOP_INSET_PX) }; + const viewport = this.page.viewportSize() ?? { height: 0, width: 0 }; + const middle = { x: viewport.width / 2, y: viewport.height / 2 }; + + await this.pinch(at, scale, 1400); + + // One near the bottom edge overshoots its ratio, and a pinch there again misses too. A small + // pinch from the middle evens it out, drawing the detail toward the middle, so it stays in view + const reached = await this.pageScale(); + + if (Math.abs(reached / scale - 1) > 0.03) await this.pinch(middle, scale / reached, 1400); + + await this.liftFingers(); + await this.pause(ms); + + // back to exactly 1x, which the next gesture needs, in one quick pinch from the middle of + // the viewport, and another for what is left when one lands short + for (let tries = 0; ; tries++) { + const current = await this.pageScale(); + + if (current <= 1.001) break; + + if (tries === 4) throw new Error(`demo: zoom stuck at ${current.toFixed(3)}x`); + + await this.pinch(middle, 1 / current, 1800); + } + + await this.liftFingers(); + await this.pause(200); + } + + private between(min: number, max: number): number { + return min + (max - min) * this.random(); + } + + private async center(target: Locator): Promise<{ x: number; y: number }> { + const box = await target.boundingBox(); + + if (!box) throw new Error(`demo: ${target} has no bounding box`); + + return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + } + + /** A pinch that reaches the screen's edge can end without its last touchend reaching the page. */ + private async liftFingers(): Promise { + await this.page.evaluate(() => + document + .querySelectorAll('.demo-finger') + .forEach((finger) => (finger.style.opacity = '0')) + ); + } + + private pageScale(): Promise { + return this.page.evaluate(() => window.visualViewport?.scale ?? 1); + } + + /** + * A two-finger pinch by `factor` around `at`, in viewport pixels. While zoomed in, the gesture + * reads its position in page-scale units, so it is divided by the scale the page is at. + */ + private async pinch(at: { x: number; y: number }, factor: number, speed: number): Promise { + const cdp = await this.session(); + const current = await this.pageScale(); + + await cdp.send('Input.synthesizePinchGesture', { + gestureSourceType: 'touch', + relativeSpeed: speed, + scaleFactor: factor, + x: at.x / current, + y: at.y / current + }); + } + + private async session(): Promise { + this.cdp ??= await this.page.context().newCDPSession(this.page); + + return this.cdp; + } +} + +interface DemoOptions { + /** Path the demo starts from. */ + startPath: string; + /** Extra UI settings merged into the seeded config. */ + settings: Record; + /** + * Conversations imported before the demo starts: built-in traces from + * fixtures/traces.ts, or `.jsonl` files from DEMO_TRACES_DATASET. + */ + traces: string[]; +} + +/** Reads a trace by name, generating built-in ones and downloading the rest. */ +function loadTrace(name: string, dir: string): string { + const generated = generateTrace(name); + + if (generated) return generated; + + const dataset = process.env.DEMO_TRACES_DATASET; + + if (!dataset) + throw new Error(`demo: no built-in trace "${name}" and DEMO_TRACES_DATASET is not set`); + + execFileSync( + 'hf', + ['download', dataset, `${name}.jsonl`, '--repo-type', 'dataset', '--local-dir', dir], + { stdio: 'inherit' } + ); + + return readFileSync(join(dir, `${name}.jsonl`), 'utf8'); +} + +/** Imports conversations through Settings > Import/Export, like a user would. */ +async function importTraces(page: Page, file: string): Promise { + // the expanded sidebar adds the shortcut to the name + const settings = page.getByRole('button', { name: /^Settings(\s|$)/ }).first(); + const reachable = (target: Locator) => + target.click({ timeout: 2000, trial: true }).then( + () => true, + () => false + ); + // a collapsed sidebar can sit under the page: on a phone the page's menu key slides it aside, + // on a desktop its logo expands it + const keys = [ + page.locator('xpath=//button[@aria-label="Expand navigation"][not(ancestor::aside)]'), + page.locator('aside button[aria-label="Expand navigation"]') + ]; + + for (const key of keys) { + if (await reachable(settings)) break; + + if ((await key.count()) > 0 && (await reachable(key.first()))) await key.first().click(); + } + + await settings.click(); + + const dialog = page.getByRole('dialog'); + + await dialog.getByText('Import/Export', { exact: true }).filter({ visible: true }).click(); + + const chooser = page.waitForEvent('filechooser'); + + await dialog.getByRole('button', { name: 'Import conversations' }).click(); + await (await chooser).setFiles(file); + await page.getByRole('button', { name: /^Import \(\d+\)$/ }).click(); + await page.getByRole('heading', { name: /^Imported \d+ conversation/ }).waitFor(); + + // the app remembers the sidebar the import expanded; a demo starts with it collapsed, as on a + // first visit + await page.evaluate(() => { + for (const key of Object.keys(localStorage)) { + if (key.endsWith('.sidebarExpanded')) localStorage.removeItem(key); + } + }); +} + +export const test = base.extend({ + demo: async ({ page }, use, testInfo) => { + await use(new Demo(page, `${testInfo.file}:${testInfo.title}`)); + }, + page: async ({ page, settings, startPath, traces }, use, testInfo) => { + await page.addInitScript(installCursorOverlay); + await page.addInitScript(seedSettings, { + mcpServers: JSON.stringify(demoMcpServerSettings()), + ...settings + }); + + if (traces.length > 0) { + const dir = testInfo.outputPath('traces'); + const file = join(dir, 'import.jsonl'); + + mkdirSync(dir, { recursive: true }); + writeFileSync(file, traces.map((name) => loadTrace(name, dir)).join('\n')); + await page.goto('/'); + await page + .getByRole('textbox', { name: 'Type a message...' }) + .waitFor({ timeout: APP_READY_TIMEOUT_MS }); + await importTraces(page, file); + } + + await page.goto(startPath); + + // a hash route after the import doesn't reload, which the conversations list needs + if (traces.length > 0) await page.reload(); + + // a fresh checkout's dev server optimizes its dependencies on the first load and reloads the + // page, which can take well past an action's timeout; the recording has not started yet + await page + .getByRole('textbox', { name: 'Type a message...' }) + .waitFor({ timeout: APP_READY_TIMEOUT_MS }); + + const viewport = page.viewportSize(); + + if (viewport) await page.mouse.move(viewport.width / 2, viewport.height / 2); + + // let MCP connections and fonts settle before the recording starts + await page.waitForTimeout(1500); + + const recorder = new ScreencastRecorder(testInfo.outputPath('frames')); + + await recorder.start(page); + await use(page); + await recorder.stop(); + await page.close(); + + if (testInfo.status !== testInfo.expectedStatus) return; + + const file = basename(testInfo.file).replace(/\.demo\.ts$/, ''); + const name = testInfo.title === 'demo' ? file : `${file}--${slugify(testInfo.title)}`; + const mp4 = resolve(VIDEOS_DIR, `${name}.mp4`); + + mkdirSync(VIDEOS_DIR, { recursive: true }); + recorder.encode(mp4); + // the CLI reads which videos a script made, in order, with their titles + appendFileSync( + resolve(VIDEOS_DIR, `${file}.videos.jsonl`), + `${JSON.stringify({ mp4, title: testInfo.title })}\n` + ); + console.log(`demo video: ${mp4}`); + }, + + settings: [{}, { option: true }], + + startPath: ['/', { option: true }], + + traces: [[], { option: true }] +}); + +export { expect } from '@playwright/test'; + +function slugify(text: string): string { + return text + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-|-$/g, ''); +} diff --git a/tools/ui/demo/playwright.config.ts b/tools/ui/demo/playwright.config.ts new file mode 100644 index 000000000000..2ac9c4c4bd24 --- /dev/null +++ b/tools/ui/demo/playwright.config.ts @@ -0,0 +1,87 @@ +import { DEMO_MCP_PORT } from './fixtures/demo-config.ts'; +import { defineConfig, devices } from '@playwright/test'; +import { existsSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const ENV_FILE = resolve(import.meta.dirname, '../.env'); + +if (existsSync(ENV_FILE)) process.loadEnvFile(ENV_FILE); + +/** + * Demo video recordings. + * + * Env (also read from tools/ui/.env): + * DEMO_UI_DIR tools/ui checkout to record (default: this one), e.g. a PR worktree + * DEMO_UI_PORT port for that checkout's vite dev server (default: 5180) + * DEMO_SERVER_ORIGIN llama-server the UI proxies to (default: http://localhost:9931) + * DEMO_LLAMA_SERVER llama-server binary started when none runs there (default: build/bin) + * DEMO_COLOR_SCHEME light | dark (default: light) + */ + +// Playwright clears outputDir when a run starts, so concurrent recordings each get +// their own; workers inherit the env, so they resolve the same directory +process.env.DEMO_RUN_ID ??= String(process.pid); + +const UI_DIR = resolve(process.env.DEMO_UI_DIR ?? resolve(import.meta.dirname, '..')); +const UI_PORT = Number(process.env.DEMO_UI_PORT ?? 5180); +const SERVER_ORIGIN = process.env.DEMO_SERVER_ORIGIN ?? 'http://localhost:9931'; +const SERVER_PORT = Number(new URL(SERVER_ORIGIN).port || 80); +const LLAMA_SERVER = + process.env.DEMO_LLAMA_SERVER ?? resolve(import.meta.dirname, '../../../build/bin/llama-server'); + +export const VIEWPORT = { height: 900, width: 1440 }; +/** Device pixel ratio the demos render and record at. */ +const SCALE = Number(process.env.DEMO_SCALE ?? 2); + +export default defineConfig({ + expect: { + timeout: 10000 + }, + fullyParallel: false, + outputDir: `test-results/${process.env.DEMO_RUN_ID}`, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'], deviceScaleFactor: SCALE, viewport: VIEWPORT } + } + ], + reporter: 'line', + retries: 0, + testDir: '.', + testMatch: ['**/*.demo.ts'], + timeout: 120000, + use: { + actionTimeout: 10000, + baseURL: `http://localhost:${UI_PORT}`, + colorScheme: process.env.DEMO_COLOR_SCHEME === 'dark' ? 'dark' : 'light', + // headless Chromium screencasts at 1x unless the scale is forced at launch; the sandbox + // is on since a PR demo runs that PR's code (Playwright turns it off by default). Not the + // installed Chrome: its headless mode screencasts a window that ignores the viewport + launchOptions: { args: [`--force-device-scale-factor=${SCALE}`], chromiumSandbox: true }, + // recorded through the screencast in recorder.ts, at the device pixel ratio + video: 'off', + viewport: VIEWPORT + }, + webServer: [ + { + command: `node ${resolve(import.meta.dirname, 'fixtures/mcp-servers.ts')}`, + reuseExistingServer: true, + url: `http://localhost:${DEMO_MCP_PORT}/health` + }, + { + command: `${LLAMA_SERVER} --host 127.0.0.1 --port ${SERVER_PORT} --tools all`, + reuseExistingServer: true, + timeout: 60000, + url: `${SERVER_ORIGIN}/health` + }, + { + command: `npx vite dev --port ${UI_PORT} --strictPort`, + cwd: UI_DIR, + env: { VITE_PUBLIC_SERVER_ORIGIN: SERVER_ORIGIN }, + reuseExistingServer: true, + timeout: 120000, + url: `http://localhost:${UI_PORT}` + } + ], + workers: 1 +}); diff --git a/tools/ui/demo/recorder.ts b/tools/ui/demo/recorder.ts new file mode 100644 index 000000000000..3d1faa9a4765 --- /dev/null +++ b/tools/ui/demo/recorder.ts @@ -0,0 +1,109 @@ +/** + * Records a page through Chromium's screencast instead of Playwright's video, + * which encodes VP8 at about 1 Mbit/s at 1x scale. Frames arrive at the page's + * device pixel ratio, each with its timestamp, and are encoded once into an + * H.264 mp4 (what GitHub accepts in PR descriptions). + */ + +import type { CDPSession, Page } from '@playwright/test'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +const FPS = 60; +const JPEG_QUALITY = 92; + +interface Frame { + file: string; + /** Seconds, on the same clock as Date.now(). */ + time: number; +} + +export class ScreencastRecorder { + private readonly dir: string; + private readonly frames: Frame[] = []; + private readonly pending: Promise[] = []; + private session: CDPSession | undefined; + private stoppedAt = 0; + + constructor(dir: string) { + this.dir = dir; + } + + /** Encodes the frames, each held until the next one arrived (the screencast only sends changes). */ + encode(mp4: string): void { + if (this.frames.length === 0) throw new Error('demo: the screencast captured no frames'); + + const list = this.frames.flatMap((frame, i) => { + const end = this.frames[i + 1]?.time ?? this.stoppedAt; + + return [`file '${frame.file}'`, `duration ${Math.max(end - frame.time, 1 / FPS).toFixed(4)}`]; + }); + const listFile = join(this.dir, 'frames.txt'); + + // the concat demuxer ignores the last entry's duration unless the file is repeated + writeFileSync(listFile, [...list, `file '${this.frames.at(-1)!.file}'`, ''].join('\n')); + execFileSync( + 'ffmpeg', + [ + '-y', + '-loglevel', + 'error', + '-f', + 'concat', + '-safe', + '0', + '-i', + listFile, + '-vf', + `fps=${FPS},scale=trunc(iw/2)*2:trunc(ih/2)*2`, + '-c:v', + 'libx264', + '-preset', + 'slow', + '-crf', + '18', + '-pix_fmt', + 'yuv420p', + '-movflags', + '+faststart', + mp4 + ], + { stdio: 'inherit' } + ); + rmSync(this.dir, { force: true, recursive: true }); + } + + async start(page: Page): Promise { + const size = page.viewportSize(); + const scale = await page.evaluate(() => window.devicePixelRatio); + + rmSync(this.dir, { force: true, recursive: true }); + mkdirSync(this.dir, { recursive: true }); + + this.session = await page.context().newCDPSession(page); + this.session.on('Page.screencastFrame', ({ data, metadata, sessionId }) => { + const file = join(this.dir, `${String(this.frames.length).padStart(6, '0')}.jpg`); + + // acknowledged first: Chromium sends the next frame only after the ack + this.session?.send('Page.screencastFrameAck', { sessionId }).catch(() => {}); + this.frames.push({ file, time: metadata.timestamp ?? Date.now() / 1000 }); + this.pending.push(writeFile(file, Buffer.from(data, 'base64'))); + }); + await this.session.send('Page.startScreencast', { + everyNthFrame: 1, + format: 'jpeg', + maxHeight: size ? size.height * scale : undefined, + maxWidth: size ? size.width * scale : undefined, + quality: JPEG_QUALITY + }); + } + + async stop(): Promise { + this.stoppedAt = Date.now() / 1000; + await this.session?.send('Page.stopScreencast').catch(() => {}); + await this.session?.detach().catch(() => {}); + await Promise.all(this.pending); + } +} diff --git a/tools/ui/demo/storage.ts b/tools/ui/demo/storage.ts new file mode 100644 index 000000000000..def006d7b8df --- /dev/null +++ b/tools/ui/demo/storage.ts @@ -0,0 +1,92 @@ +/** + * Storage targets for demo scripts and videos, picked by the scheme of the + * DEMO_STORAGE URI: + * + * hf://buckets//[/prefix] Hugging Face bucket, via the `hf` CLI login + * file:///some/dir local or mounted directory + */ + +import { execFileSync } from 'node:child_process'; +import { copyFileSync, cpSync, existsSync, mkdirSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export interface DemoStorage { + /** Copies every file under remote `dir` into local `target`; returns false when it has none. */ + downloadDir(dir: string, target: string): boolean; + /** Copies local `file` to remote `key`. */ + upload(file: string, key: string): void; +} + +function joinKey(base: string, key: string): string { + return `${base.replace(/\/+$/, '')}/${key}`; +} + +// plain fields: the CLI runs under Node's type stripping, which has no parameter properties +class HfStorage implements DemoStorage { + private readonly base: string; + + constructor(base: string) { + this.base = base; + } + + downloadDir(dir: string, target: string): boolean { + mkdirSync(target, { recursive: true }); + + try { + execFileSync('hf', ['buckets', 'sync', joinKey(this.base, dir), target], { + stdio: 'inherit' + }); + } catch { + return false; + } + + return true; + } + + upload(file: string, key: string): void { + execFileSync('hf', ['buckets', 'cp', file, joinKey(this.base, key)], { stdio: 'inherit' }); + } +} + +class FileStorage implements DemoStorage { + private readonly root: string; + + constructor(root: string) { + this.root = root; + } + + downloadDir(dir: string, target: string): boolean { + const source = join(this.root, dir); + + if (!existsSync(source)) return false; + + cpSync(source, target, { recursive: true }); + + return true; + } + + upload(file: string, key: string): void { + const target = join(this.root, key); + + mkdirSync(dirname(target), { recursive: true }); + copyFileSync(file, target); + } +} + +export function getDemoStorage(uri = process.env.DEMO_STORAGE): DemoStorage | null { + if (!uri) return null; + + if (uri.startsWith('hf://')) return new HfStorage(uri); + + if (uri.startsWith('file://')) return new FileStorage(fileURLToPath(uri)); + + throw new Error(`DEMO_STORAGE: unsupported scheme in "${uri}" (use hf:// or file://)`); +} + +/** Public URL of a stored key, when DEMO_STORAGE_PUBLIC_URL is set. */ +export function getPublicUrl(key: string): string | null { + const base = process.env.DEMO_STORAGE_PUBLIC_URL; + + return base ? joinKey(base, key) : null; +} diff --git a/tools/ui/package.json b/tools/ui/package.json index e87d40aa0279..c29469d2d886 100644 --- a/tools/ui/package.json +++ b/tools/ui/package.json @@ -18,6 +18,11 @@ "test": "npm run test:ui -- --run && npm run test:client -- --run && npm run test:unit -- --run && npm run test:e2e", "test:e2e": "playwright test", "test:e2e:pwa": "playwright test tests/e2e/pwa.e2e.ts", + "demo:codegen": "playwright codegen --target=playwright-test --viewport-size=1440,900 --output=demo/recording.demo.ts http://localhost:5180/", + "demo:docs": "node demo/cli.ts docs", + "demo:pr": "node demo/cli.ts pr", + "demo:record": "playwright test --config=demo/playwright.config.ts", + "demo:traces": "node demo/cli.ts traces", "test:client": "vitest --project=client", "test:unit": "vitest --project=unit", "test:unit:pwa": "vitest --project=unit --run tests/unit/pwa.spec.ts", From 749db7f034d43d6de01a01f6d8c872d02afa6c7b Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 3 Nov 2026 12:47:23 +0100 Subject: [PATCH 024/286] qwen4exp: write the conv state tail without an intermediate cont (#31443) build_conv_state_at makes a contiguous copy of the conv input tail before writing it into the recurrent state, so every GDN layer issues two device copies per generated token where one is enough. On Qwen3.8-Flash-Next (36 GDN layers) that is 72 copies per token, now 36. Speculative decoding adds one more per rollback slot. ggml_cpy reads the strided view directly, as the shared build_conv_state already does, so the extra copy is dropped. --- src/models/qwen4exp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/models/qwen4exp.cpp b/src/models/qwen4exp.cpp index 60665381c660..e4848cb77f26 100644 --- a/src/models/qwen4exp.cpp +++ b/src/models/qwen4exp.cpp @@ -1443,7 +1443,7 @@ ggml_tensor * llama_model_qwen4exp::graph::build_conv_state_at( conv_states_all->nb[1], (slot * mem_size + kv_head) * row_size); - ggml_build_forward_expand(gf, ggml_cpy(ctx0, ggml_cont(ctx0, tail), dst)); + ggml_build_forward_expand(gf, ggml_cpy(ctx0, tail, dst)); } return conv_input; From 28b812d6cb9cfe569d368c2774c3886ad4fe10e8 Mon Sep 17 00:00:00 2001 From: Prabhsimran Singh Date: Tue, 3 Nov 2026 18:57:05 +0530 Subject: [PATCH 025/286] mtmd : add Nemotron 3.5 Transcribe support (#31471) * mtmd : add Nemotron 3.5 Transcribe support Assisted-by: Claude * convert : fix vocab of draft models for Nemotron-H targets Assisted-by: Claude * convert : reject non-BF16 Transcribe checkpoints Assisted-by: Claude * mtmd : make speaker FIFO predictions local Assisted-by: Claude * mtmd : normalize the mel in encode_windows Assisted-by: Claude * convert : map the generated speaker kernel Assisted-by: Claude * mtmd : validate Nemotron Transcribe mmproj shapes Assisted-by: Claude --- conversion/__init__.py | 2 + conversion/base.py | 2 +- conversion/nemotron.py | 149 +++++++++- gguf-py/gguf/constants.py | 60 ++++ gguf-py/gguf/gguf_writer.py | 9 + gguf-py/gguf/tensor_mapping.py | 87 +++++- tools/mtmd/CMakeLists.txt | 1 + tools/mtmd/clip-impl.h | 19 ++ tools/mtmd/clip-model.h | 30 ++ tools/mtmd/clip.cpp | 146 ++++++++- tools/mtmd/models/models.h | 16 + tools/mtmd/models/nemotron-transcribe.cpp | 343 ++++++++++++++++++++++ tools/mtmd/mtmd-audio.cpp | 49 +++- tools/mtmd/mtmd-audio.h | 9 + tools/mtmd/mtmd.cpp | 4 + 15 files changed, 907 insertions(+), 19 deletions(-) create mode 100644 tools/mtmd/models/nemotron-transcribe.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 6a34da41fe4a..dcbb97c1f85f 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -223,6 +223,7 @@ "ModernBertForSequenceClassification": "bert", "ModernBertModel": "bert", "NanbeigeForCausalLM": "nanbeige", + "Nemotron3_5TranscribeForConditionalGeneration": "nemotron", "NemotronForCausalLM": "nemotron", "NemotronHForCausalLM": "nemotron", "NemotronHPuzzleForCausalLM": "nemotron", @@ -380,6 +381,7 @@ "MiniCPMOcrForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "Mistral3Model": "llava", + "Nemotron3_5TranscribeForConditionalGeneration": "nemotron", "NemotronH_Omni_Reasoning_V3": "nemotron", "NemotronH_Nano_VL_V2": "nemotron", "MuseGlimmerForConditionalGeneration": "muse_glimmer", diff --git a/conversion/base.py b/conversion/base.py index 1c7aecec877c..fd880bbaba86 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -3386,7 +3386,7 @@ def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> st # Kimi-K3's text_config reports "KimiLinearForCausalLM", which is the older # Kimi-Linear-48B architecture and cannot load K3 (no attention residuals, # latent MoE, situ, ...). Route on the top-level architecture instead. - if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration"): + if model_type == ModelType.TEXT and arch in ("NEOChatModel", "StepVLForConditionalGeneration", "Sarashina2VisionForCausalLM", "Exaone4_5_ForConditionalGeneration", "Step3p7ForConditionalGeneration", "KimiK3ForConditionalGeneration", "Nemotron3_5TranscribeForConditionalGeneration"): return arch # if "architectures" is found in the sub-config, use that instead diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 371f5b616f64..066149744ca2 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -428,6 +428,9 @@ class NemotronHModel(GraniteHybridModel): is_moe: bool = False supports_mtp_export = True _experts: list[dict[str, Tensor]] | None = None + # tokens past the embedding rows (left out of the vocab), and special tokens that are printed as text + _vocab_skip: tuple[str, ...] = () + _user_defined_tokens: re.Pattern[str] | None = None _SSM_LAYER_TYPES = {"mamba", "linear_attention"} _ATTN_LAYER_TYPES = {"attention", "full_attention"} @@ -616,7 +619,11 @@ def set_vocab(self): vocab_size = -(vocab_size // -pad_vocab) * pad_vocab self.hparams["vocab_size"] = vocab_size - assert max(tokenizer.vocab.values()) < vocab_size # ty: ignore[unresolved-attribute] + # draft-model converters (DFlash, LiLiCorr) call this with their own instance, which has neither hook + vocab_skip = getattr(self, "_vocab_skip", ()) + user_defined_tokens = getattr(self, "_user_defined_tokens", None) + + assert max(id_ for tok, id_ in tokenizer.vocab.items() if tok not in vocab_skip) < vocab_size # ty: ignore[unresolved-attribute] tokpre = self.get_vocab_base_pre(tokenizer) @@ -638,7 +645,8 @@ def set_vocab(self): if previous_token != token: logger.info(f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer") - if added_tokens_decoder[i].special or self.does_token_look_special(token): + printed = user_defined_tokens is not None and user_defined_tokens.fullmatch(token) + if (added_tokens_decoder[i].special or self.does_token_look_special(token)) and not printed: toktypes.append(gguf.TokenType.CONTROL) else: token = token.replace(b"\xe2\x96\x81".decode("utf-8"), " ") # pre-normalize user-defined spaces @@ -834,3 +842,140 @@ def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Ca if item[0].startswith("mtp."): return None return super().filter_tensors(item) + + +# NeMo speechlm2 SALM checkpoint: a Nemotron-H decoder under llm.*, the speech encoder under perception.* +@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration") +@ModelBase.example("nvidia/NVIDIA-Nemotron-3.5-Transcribe-30B-A4B") +class NemotronTranscribeModel(NemotronHModel): + model_arch = gguf.MODEL_ARCH.NEMOTRON_H + # the audio placeholder (id vocab_size) has no embedding row; speaker, event and style tags print with the transcript + _vocab_skip = ("<|audio|>",) + _user_defined_tokens = re.compile(r"||") + _renames = { + "llm.model.embed_tokens.weight": "backbone.embeddings.weight", + "llm.model.norm.weight": "backbone.norm_f.weight", + "llm.lm_head.weight": "lm_head.weight", + } + # grouped experts: [n_expert, n_in, n_out] -> [n_expert, n_out, n_in] + _grouped_experts = {"gate_and_up_projs": "up_proj", "down_projs": "down_proj"} + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.startswith(("perception.", "ctc_timestamp.")) or "_extra_state" in name: + return None + if not name.startswith("llm."): + # the FP8 and NVFP4 releases store the decoder as backbone.* + raise ValueError(f"unexpected tensor {name}: only the BF16 checkpoint layout (llm.*) is supported") + if name in cls._renames: + name = cls._renames[name] + elif name.startswith("llm.model.layers."): + name = "backbone.layers." + name[len("llm.model.layers."):] + elif name.startswith("llm.mtp.layers."): + name = "mtp.layers." + name[len("llm.mtp.layers."):] + else: + raise ValueError(f"unexpected decoder tensor {name}") + return super().filter_tensors((name.replace("._fp32_params.", "."), gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + leaf = name.rsplit(".", 1)[-1] + if ".mixer.experts." in name and leaf in self._grouped_experts: + if name.startswith("mtp."): + assert self._mtp_bid is not None, f"{name}: MTP head is not exported" + bid = self._mtp_bid + assert bid is not None + data_torch = data_torch.permute(0, 2, 1).contiguous() + name = f"model.layers.{bid}.mlp.experts.{self._grouped_experts[leaf]}.weight" + yield from ModelBase.modify_tensors(self, data_torch, name, bid) + return + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Nemotron3_5TranscribeForConditionalGeneration") +class NemotronTranscribeAudioModel(MmprojModel): + # NeMo ParallelExpertEncoder: ASR and speaker transformers, speaker activity fused into the ASR states + has_vision_encoder = False + has_audio_encoder = True + + def get_audio_config(self) -> dict[str, Any] | None: + asr = self.global_config["pe_encoder_config"]["asr_encoder_cfg"] + return { + "hidden_size": asr["d_model"], + "intermediate_size": int(float(asr.get("ff_expansion", 4)) * asr["d_model"]), + "num_hidden_layers": asr["n_layers"], + "num_attention_heads": asr["n_heads"], + } + + @staticmethod + def _rope_dims(cfg: dict[str, Any]) -> int: + assert cfg.get("self_attention_model") == "rope" and cfg.get("subsampling", "feature_stacking") == "feature_stacking" + assert cfg.get("attn_mode", "full") == "full" and not cfg.get("xscaling", False) and cfg.get("pre_block_norm", True) + assert float(cfg.get("rope_base", 10000.0)) == 10000.0 + n_rot = int(cfg["d_model"] // cfg["n_heads"] * float(cfg.get("rotary_fraction", 1.0))) + assert n_rot >= 2 and n_rot % 2 == 0 + return n_rot + + def set_gguf_parameters(self): + super().set_gguf_parameters() + pe = self.global_config["pe_encoder_config"] + asr = pe["asr_encoder_cfg"] + diar = pe["diarization_model_cfg"] + spk = diar["encoder"] + assert pe.get("asr_encoder_type") == "transformer" and pe.get("speaker_feature_mode", "thresholded") == "thresholded" + assert pe.get("asr_normalize_type") == "per_feature" and pe.get("diar_normalize_type") in (None, "NA") + assert asr["subsampling_factor"] == spk["subsampling_factor"] == 8 and asr["feat_in"] == spk["feat_in"] + assert self._rope_dims(spk) == spk["d_model"] // spk["n_heads"] and not spk.get("qkv_bias", False) + assert diar.get("high_resolution", False) + # constants in clip: the ParallelExpertEncoder online settings and the Sortformer streaming config + assert (pe.get("online_inference_length", 500), pe.get("chunk_left_context", 50), pe.get("chunk_right_context", 50), + pe.get("diar_spkcache_len", 188), pe.get("diar_fifo_len", 40), pe.get("diar_spkcache_update_period", 300), + pe.get("speaker_activity_threshold", 0.5)) == (500, 50, 50, 188, 40, 300, 0.5), \ + "streaming settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp" + sm = diar["sortformer_modules"] + assert (sm["spkcache_sil_frames_per_spk"], sm["pred_score_threshold"], sm["scores_boost_latest"], sm["strong_boost_rate"], + sm["weak_boost_rate"], sm["min_pos_scores_rate"], sm["use_learnable_sil_emb"]) == (1, 0.25, 0.05, 0.75, 1.5, 0.5, True), \ + "speaker cache settings differ from the constants in tools/mtmd/models/nemotron-transcribe.cpp" + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.NEMOTRON_TRANSCRIBE) + self.gguf_writer.add_audio_num_mel_bins(asr["feat_in"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + self.gguf_writer.add_audio_subsampling_factor(asr["subsampling_factor"]) + self.gguf_writer.add_audio_rope_dims(self._rope_dims(asr)) + self.gguf_writer.add_audio_spk_block_count(spk["n_layers"]) + self.gguf_writer.add_audio_spk_head_count(spk["n_heads"]) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + # ParallelExpertEncoder._build_sinusoid_position_encoding + pe = self.global_config["pe_encoder_config"] + n_spk = int(pe["diarization_model_cfg"]["sortformer_modules"]["num_spks"]) + d = int(pe["asr_encoder_cfg"]["d_model"]) + scale = float(pe.get("spk_kernel_scale", 1.0)) + pos = torch.arange(n_spk, dtype=torch.float32)[:, None] + div = torch.exp(torch.arange(0, d, 2, dtype=torch.float32) * -(np.log(10000.0) / d)) + kernel = torch.zeros(n_spk, d, dtype=torch.float32) + kernel[:, 0::2] = torch.sin(pos * div) + kernel[:, 1::2] = torch.cos(pos * div) + # stored as [d, n_spk]: the graph multiplies it with the speaker activity directly + yield self.format_tensor_name(gguf.MODEL_TENSOR.A_FUSION_SPK_KERNEL, suffix=""), (kernel * scale).T.contiguous() + + def tensor_force_quant(self, name, new_name, bid, n_dims): + if self.match_model_tensor_name(new_name, gguf.MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, None): + return gguf.GGMLQuantizationType.F16 # ggml_conv_1d kernel + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("perception.") or "_extra_state" in name: + return None + # training-only head; the speaker model's unused front end + if name.startswith(("perception.encoder.diarization_model.sortformer_modules.hidden_to_spks.", + "perception.encoder.diarization_model.preprocessor.")): + return None + return name, gen + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name == "perception.preprocessor.featurizer.fb": + data_torch = data_torch.squeeze(0) + yield from super().modify_tensors(data_torch, name, bid) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4d057295c128..ad4e675f69a8 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -506,6 +506,11 @@ class ClipAudio: WINDOW_SIZE = "clip.audio.window_size" LOCAL_BLOCK_COUNT = "clip.audio.local_block_count" # mimo-v2.5: input_local_transformer layer count LOCAL_GROUP_SIZE = "clip.audio.local_group_size" # mimo-v2.5: input_local_transformer grouping size + ROPE_DIMS = "clip.audio.rope_dims" # nemotron-transcribe + + class Speaker: # nemotron-transcribe + BLOCK_COUNT = "clip.audio.spk.block_count" + HEAD_COUNT = "clip.audio.spk.head_count" class Attention: HEAD_COUNT = "clip.audio.attention.head_count" @@ -1411,6 +1416,24 @@ class MODEL_TENSOR(IntEnum): A_ENC_CONV_NORM_VAR = auto() # parakeet A_ENC_MEL_FILTERS = auto() # parakeet A_ENC_WINDOW = auto() # parakeet + A_ENC_ATTN_QKV = auto() # nemotron-transcribe + A_SPK_INP_PROJ = auto() # nemotron-transcribe + A_SPK_PRE_NORM = auto() # nemotron-transcribe + A_SPK_POST_NORM = auto() # nemotron-transcribe + A_SPK_ATTN_NORM = auto() # nemotron-transcribe + A_SPK_ATTN_QKV = auto() # nemotron-transcribe + A_SPK_ATTN_OUT = auto() # nemotron-transcribe + A_SPK_FFN_NORM = auto() # nemotron-transcribe + A_SPK_FFN_UP = auto() # nemotron-transcribe + A_SPK_FFN_DOWN = auto() # nemotron-transcribe + A_SPK_HEAD_PROJ = auto() # nemotron-transcribe + A_SPK_HEAD_UPSAMPLE = auto() # nemotron-transcribe + A_SPK_HEAD_FC1 = auto() # nemotron-transcribe + A_SPK_HEAD_FC2 = auto() # nemotron-transcribe + A_SPK_SIL_EMB = auto() # nemotron-transcribe + A_FUSION_ASR_NORM = auto() # nemotron-transcribe + A_FUSION_SPK_NORM = auto() # nemotron-transcribe + A_FUSION_SPK_KERNEL = auto() # nemotron-transcribe A_CTC_OUT = auto() A_CTC_OUT_MID = auto() A_ENC_ATTN_REL_POS_EMB = auto() @@ -2219,6 +2242,24 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_CONV_NORM_VAR: "a.blk.{bid}.conv_norm_var", MODEL_TENSOR.A_ENC_MEL_FILTERS: "a.mel_filters", MODEL_TENSOR.A_ENC_WINDOW: "a.window", + MODEL_TENSOR.A_ENC_ATTN_QKV: "a.blk.{bid}.attn_qkv", + MODEL_TENSOR.A_SPK_INP_PROJ: "a.spk.input_projection", + MODEL_TENSOR.A_SPK_PRE_NORM: "a.spk.pre_ln", + MODEL_TENSOR.A_SPK_POST_NORM: "a.spk.post_ln", + MODEL_TENSOR.A_SPK_ATTN_NORM: "a.spk.blk.{bid}.ln1", + MODEL_TENSOR.A_SPK_ATTN_QKV: "a.spk.blk.{bid}.attn_qkv", + MODEL_TENSOR.A_SPK_ATTN_OUT: "a.spk.blk.{bid}.attn_out", + MODEL_TENSOR.A_SPK_FFN_NORM: "a.spk.blk.{bid}.ln2", + MODEL_TENSOR.A_SPK_FFN_UP: "a.spk.blk.{bid}.ffn_up", + MODEL_TENSOR.A_SPK_FFN_DOWN: "a.spk.blk.{bid}.ffn_down", + MODEL_TENSOR.A_SPK_HEAD_PROJ: "a.spk.head.proj", + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: "a.spk.head.upsample", + MODEL_TENSOR.A_SPK_HEAD_FC1: "a.spk.head.fc1", + MODEL_TENSOR.A_SPK_HEAD_FC2: "a.spk.head.fc2", + MODEL_TENSOR.A_SPK_SIL_EMB: "a.spk.head.sil_emb", + MODEL_TENSOR.A_FUSION_ASR_NORM: "a.fusion.asr_norm", + MODEL_TENSOR.A_FUSION_SPK_NORM: "a.fusion.spk_norm", + MODEL_TENSOR.A_FUSION_SPK_KERNEL: "a.fusion.spk_kernel", MODEL_TENSOR.A_CTC_OUT: "a.enc_ctc_out", MODEL_TENSOR.A_CTC_OUT_MID: "a.enc_ctc_out_mid", MODEL_TENSOR.A_ENC_ATTN_REL_POS_EMB: "a.blk.{bid}.attn_rel_pos_emb", @@ -2587,6 +2628,24 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.A_ENC_CONV_NORM_VAR, MODEL_TENSOR.A_ENC_MEL_FILTERS, MODEL_TENSOR.A_ENC_WINDOW, + MODEL_TENSOR.A_ENC_ATTN_QKV, + MODEL_TENSOR.A_SPK_INP_PROJ, + MODEL_TENSOR.A_SPK_PRE_NORM, + MODEL_TENSOR.A_SPK_POST_NORM, + MODEL_TENSOR.A_SPK_ATTN_NORM, + MODEL_TENSOR.A_SPK_ATTN_QKV, + MODEL_TENSOR.A_SPK_ATTN_OUT, + MODEL_TENSOR.A_SPK_FFN_NORM, + MODEL_TENSOR.A_SPK_FFN_UP, + MODEL_TENSOR.A_SPK_FFN_DOWN, + MODEL_TENSOR.A_SPK_HEAD_PROJ, + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE, + MODEL_TENSOR.A_SPK_HEAD_FC1, + MODEL_TENSOR.A_SPK_HEAD_FC2, + MODEL_TENSOR.A_SPK_SIL_EMB, + MODEL_TENSOR.A_FUSION_ASR_NORM, + MODEL_TENSOR.A_FUSION_SPK_NORM, + MODEL_TENSOR.A_FUSION_SPK_KERNEL, MODEL_TENSOR.A_MM_INP_PROJ, MODEL_TENSOR.A_MM_SOFT_EMB_NORM, MODEL_TENSOR.A_MM_EMBEDDING, @@ -6740,6 +6799,7 @@ class VisionProjectorType: MINICPMV4_7 = "minicpmv4_7" MINICPM_OCR_1_0 = "minicpm_ocr_1_0" GRANITE_SPEECH = "granite_speech" # audio + NEMOTRON_TRANSCRIBE = "nemotron_transcribe" # audio MIMOVL = "mimovl" MIMO_AUDIO = "mimo_audio" GRANITE4_VISION = "granite4_vision" diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index a0800e24c0a6..56e8938c4f63 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1712,6 +1712,15 @@ def add_audio_projector_downsample_rate(self, value: int) -> None: def add_audio_projector_head_count(self, value: int) -> None: self.add_uint32(Keys.ClipAudio.Projector.HEAD_COUNT, value) + def add_audio_rope_dims(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.ROPE_DIMS, value) + + def add_audio_spk_block_count(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.Speaker.BLOCK_COUNT, value) + + def add_audio_spk_head_count(self, value: int) -> None: + self.add_uint32(Keys.ClipAudio.Speaker.HEAD_COUNT, value) + # audio generation (mmproj) def add_clip_gen_audio_projector_type(self, value: str) -> None: diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index d06dc4ec5fdc..2602bfb61b3d 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -2532,6 +2532,7 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_INP_PROJ: ( "conformer.subsample_conv_projection.input_proj_linear", # gemma4 + "perception.encoder.asr_encoder.pre_encode.proj", # nemotron-transcribe (feature stacking) "encoder.input_linear", ), @@ -2546,13 +2547,16 @@ class TensorNameMap: "speaker_encoder.mfa.conv", # qwen3tts speaker encoder: multi-layer feature aggregation ), - MODEL_TENSOR.A_PRE_NORM: (), + MODEL_TENSOR.A_PRE_NORM: ( + "perception.encoder.asr_encoder.embed_norm", # nemotron-transcribe + ), MODEL_TENSOR.A_POST_NORM: ( "audio_tower.layer_norm", # ultravox "audio_tower.norm", # voxtral realtime "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note "audio_tower.ln_post", # qwen2omni + "perception.encoder.asr_encoder.final_norm", # nemotron-transcribe "encoder.layer_norm", # mimo-audio-tokenizer "encoder.lfm.lfm.embedding_norm", # lfm3a ), @@ -2595,10 +2599,12 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_ATTN_Q_NORM: ( "encoder.lfm.lfm.layers.{bid}.self_attn.q_layernorm", # lfm3a + "perception.encoder.asr_encoder.layers.{bid}.attn.q_norm", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_ATTN_K_NORM: ( "encoder.lfm.lfm.layers.{bid}.self_attn.k_layernorm", # lfm3a + "perception.encoder.asr_encoder.layers.{bid}.attn.k_norm", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_ATTN_K_REL: ( @@ -2629,6 +2635,7 @@ class TensorNameMap: "encoder.lfm.lfm.layers.{bid}.operator_norm", # lfm3a "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.norm1", # nemotron-transcribe "encoder.layers.{bid}.attn.pre_norm", # granite_speech "encoder.layers.{bid}.self_attn_layer_norm", # mimo-audio-tokenizer ), @@ -2642,6 +2649,7 @@ class TensorNameMap: "conformer.layers.{bid}.attention.post", # gemma3n "conformer.layers.{bid}.self_attn.post", # gemma4 "sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe "encoder.layers.{bid}.attn.to_out", # granite_speech "encoder.layers.{bid}.self_attn.out_proj", # mimo-audio-tokenizer ), @@ -2653,6 +2661,7 @@ class TensorNameMap: "encoder.lfm.lfm.layers.{bid}.ffn_norm", # lfm3a "conformer.layers.{bid}.attention.post_norm", # gemma3n "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.norm2", # nemotron-transcribe "encoder.layers.{bid}.post_norm", # granite_speech "encoder.layers.{bid}.final_layer_norm", # mimo-audio-tokenizer ), @@ -2683,6 +2692,7 @@ class TensorNameMap: "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4 "sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe "encoder.layers.{bid}.ff1.up_proj", # granite_speech "encoder.layers.{bid}.fc1", # mimo-audio-tokenizer ), @@ -2702,6 +2712,7 @@ class TensorNameMap: "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4 "sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet + "perception.encoder.asr_encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe "encoder.layers.{bid}.ff1.down_proj", # granite_speech "encoder.layers.{bid}.fc2", # mimo-audio-tokenizer ), @@ -2792,6 +2803,7 @@ class TensorNameMap: "audio_tower.proj", # qwen2omni "model.audio_tower.output_proj", # gemma4 "speaker_encoder.fc", # qwen3tts speaker encoder: final speaker embedding projection + "perception.proj", # nemotron-transcribe ), MODEL_TENSOR.A_MM_NORM_PRE: ( @@ -2914,12 +2926,85 @@ class TensorNameMap: MODEL_TENSOR.A_ENC_MEL_FILTERS: ( "sound_encoder.encoder.feature_extractor.featurizer.fb", # parakeet + "perception.preprocessor.featurizer.fb", # nemotron-transcribe ), MODEL_TENSOR.A_ENC_WINDOW: ( "sound_encoder.encoder.feature_extractor.featurizer.window", # parakeet + "perception.preprocessor.featurizer.window", # nemotron-transcribe + ), + + MODEL_TENSOR.A_ENC_ATTN_QKV: ( + "perception.encoder.asr_encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_INP_PROJ: ( + "perception.encoder.diarization_model.encoder.pre_encode.proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_PRE_NORM: ( + "perception.encoder.diarization_model.encoder.embed_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_POST_NORM: ( + "perception.encoder.diarization_model.encoder.final_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_NORM: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.norm1", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_QKV: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.attn.w_qkv", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_ATTN_OUT: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.attn.out_proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_NORM: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.norm2", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_UP: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.0", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_FFN_DOWN: ( + "perception.encoder.diarization_model.encoder.layers.{bid}.ffn.net.3", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_PROJ: ( + "perception.encoder.diarization_model.sortformer_modules.encoder_proj", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_UPSAMPLE: ( + "perception.encoder.diarization_model.sortformer_modules.subpixel_upsample", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_HEAD_FC1: ( + "perception.encoder.diarization_model.sortformer_modules.first_hidden_to_hidden", # nemotron-transcribe ), + MODEL_TENSOR.A_SPK_HEAD_FC2: ( + "perception.encoder.diarization_model.sortformer_modules.single_hidden_to_spks", # nemotron-transcribe + ), + + MODEL_TENSOR.A_SPK_SIL_EMB: ( + "perception.encoder.diarization_model.sortformer_modules.learnable_sil_emb", # nemotron-transcribe + ), + + MODEL_TENSOR.A_FUSION_ASR_NORM: ( + "perception.encoder.asr_norm", # nemotron-transcribe + ), + + MODEL_TENSOR.A_FUSION_SPK_NORM: ( + "perception.encoder.diar_norm", # nemotron-transcribe + ), + + # generated in conversion code (nemotron-transcribe) + MODEL_TENSOR.A_FUSION_SPK_KERNEL: (), + MODEL_TENSOR.A_MM_EMBEDDING: ( "model.embed_audio.embedding", # gemma3n ), diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index f1ec3c456645..e514de60bdf9 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -47,6 +47,7 @@ add_library(mtmd models/kimik25.cpp models/nemotron-v2-vl.cpp models/nemotron-omni.cpp + models/nemotron-transcribe.cpp models/muse-glimmer.cpp models/muse-spark-audio.cpp models/llama4.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 0504e8fdcc4c..52d5fffe33f7 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -113,6 +113,11 @@ #define KEY_GEN_AUDIO_PENALTY_LAST_N "clip.gen.audio.sampling.penalty_last_n" #define KEY_AUDIO_SUBSMPL_FACTOR "clip.audio.subsampling_factor" +// nemotron transcribe +#define KEY_A_ROPE_DIMS "clip.audio.rope_dims" +#define KEY_A_SPK_N_BLOCK "clip.audio.spk.block_count" +#define KEY_A_SPK_N_HEAD "clip.audio.spk.head_count" + // // tensor name constants // @@ -469,6 +474,13 @@ #define TN_CONV_NORM_MEAN "%s.blk.%d.conv_norm_mean" #define TN_CONV_NORM_VAR "%s.blk.%d.conv_norm_var" +// nemotron transcribe +#define TN_A_SPK_INP_PROJ "a.spk.input_projection.%s" +#define TN_A_SPK_HEAD "a.spk.head.%s.%s" // proj, upsample, fc1, fc2 +#define TN_A_SPK_SIL_EMB "a.spk.head.sil_emb" +#define TN_A_FUSION_NORM "a.fusion.%s.%s" // asr_norm, spk_norm +#define TN_A_FUSION_KERNEL "a.fusion.spk_kernel" + // align x to upper multiple of n #define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n)) @@ -487,6 +499,11 @@ struct clip_enc_state { // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place std::vector slots; + + // nemotron transcribe: the window's normalized log-mel, cache and FIFO rows in; its speaker rows and predictions out + std::vector mel_norm; + std::vector spk_prev; + std::vector spk_chunk, spk_preds; }; // per-sequence state of audio generation, see clip_gen_state_init() @@ -567,6 +584,7 @@ enum projector_type { PROJECTOR_TYPE_MINICPMV4_7, PROJECTOR_TYPE_MINICPM_OCR_1_0, PROJECTOR_TYPE_GRANITE_SPEECH, + PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, PROJECTOR_TYPE_MIMOVL, PROJECTOR_TYPE_MINIMAX_M3, PROJECTOR_TYPE_GRANITE4_VISION, @@ -647,6 +665,7 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_MINICPMV4_7, "minicpmv4_7"}, { PROJECTOR_TYPE_MINICPM_OCR_1_0, "minicpm_ocr_1_0"}, { PROJECTOR_TYPE_GRANITE_SPEECH, "granite_speech"}, + { PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE, "nemotron_transcribe"}, { PROJECTOR_TYPE_MIMOVL, "mimovl"}, { PROJECTOR_TYPE_MINIMAX_M3, "minimax_m3"}, { PROJECTOR_TYPE_GRANITE4_VISION, "granite4_vision"}, diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index 2810a5cc665d..c42f417c83ca 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -165,6 +165,12 @@ struct clip_hparams { std::vector mel_filters; std::vector window; + // nemotron transcribe + int32_t audio_rope_dims = 0; + int32_t spk_n_embd = 0; + int32_t spk_n_layer = 0; + int32_t spk_n_head = 0; + // mimo-audio-tokenizer: residual vector quantizer int32_t rvq_num_quantizers = 0; std::vector rvq_codebook_size; // per-quantizer bin count (ragged, e.g. 1024/1024/256/128x17) @@ -292,6 +298,8 @@ struct clip_layer { ggml_tensor * k_norm = nullptr; ggml_tensor * q_norm = nullptr; + ggml_tensor * k_norm_b = nullptr; + ggml_tensor * q_norm_b = nullptr; // lfm3: per-head sigmoid gate on the attention output ggml_tensor * attn_gate_w = nullptr; @@ -926,6 +934,28 @@ struct clip_model { std::vector sam_layers; + // nemotron transcribe: speaker expert, Sortformer head, fusion + std::vector spk_layers; + ggml_tensor * spk_inp_proj_w = nullptr; + ggml_tensor * spk_pre_ln_w = nullptr; + ggml_tensor * spk_pre_ln_b = nullptr; + ggml_tensor * spk_post_ln_w = nullptr; + ggml_tensor * spk_post_ln_b = nullptr; + ggml_tensor * spk_head_proj_w = nullptr; + ggml_tensor * spk_head_proj_b = nullptr; + ggml_tensor * spk_head_up_w = nullptr; + ggml_tensor * spk_head_up_b = nullptr; + ggml_tensor * spk_head_fc1_w = nullptr; + ggml_tensor * spk_head_fc1_b = nullptr; + ggml_tensor * spk_head_fc2_w = nullptr; + ggml_tensor * spk_head_fc2_b = nullptr; + ggml_tensor * fusion_asr_norm_w = nullptr; + ggml_tensor * fusion_asr_norm_b = nullptr; + ggml_tensor * fusion_spk_norm_w = nullptr; + ggml_tensor * fusion_spk_norm_b = nullptr; + ggml_tensor * fusion_spk_kernel = nullptr; + std::vector spk_sil_emb; + // deepseek-ocr-2 ggml_tensor * resample_query_768 = nullptr; ggml_tensor * resample_query_1024 = nullptr; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9a043d7f9ca3..f2c5c31b83c1 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -405,13 +405,13 @@ ggml_tensor * clip_graph::build_vit( if (layer.q_norm) { GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]); - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); cb(Qcur, "Qcur_norm", il); } if (layer.k_norm) { GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]); - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); cb(Kcur, "Kcur_norm", il); } @@ -437,11 +437,11 @@ ggml_tensor * clip_graph::build_vit( if (!norm_per_head) { if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); cb(Qcur, "Qcur_norm", il); } if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); cb(Kcur, "Kcur_norm", il); } } @@ -452,11 +452,11 @@ ggml_tensor * clip_graph::build_vit( if (norm_per_head) { if (layer.q_norm) { - Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il); + Qcur = build_norm(Qcur, layer.q_norm, layer.q_norm_b, norm_t, eps, il); cb(Qcur, "Qcur_norm_per_head", il); } if (layer.k_norm) { - Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il); + Kcur = build_norm(Kcur, layer.k_norm, layer.k_norm_b, norm_t, eps, il); cb(Kcur, "Kcur_norm_per_head", il); } } @@ -1242,6 +1242,10 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const { builder = std::make_unique(ctx, img); } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + builder = std::make_unique(ctx, img, params ? params->enc_state : nullptr); + } break; case PROJECTOR_TYPE_GRANITE4_VISION: { builder = std::make_unique(ctx, img); @@ -1623,6 +1627,27 @@ struct clip_model_loader { hparams.audio_window_len = 400; hparams.audio_hop_len = 160; } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + get_u32(KEY_AUDIO_SUBSMPL_FACTOR, hparams.subsampling_factor); + get_u32(KEY_A_ROPE_DIMS, hparams.audio_rope_dims); + get_u32(KEY_A_SPK_N_BLOCK, hparams.spk_n_layer); + get_u32(KEY_A_SPK_N_HEAD, hparams.spk_n_head); + if (hparams.subsampling_factor != 8) { + throw std::runtime_error(string_format("%s: unsupported subsampling factor %d\n", __func__, hparams.subsampling_factor)); + } + if (hparams.spk_n_layer <= 0 || hparams.spk_n_layer > (int) gguf_get_n_tensors(ctx_gguf.get())) { + throw std::runtime_error(string_format("%s: invalid speaker block count %d\n", __func__, hparams.spk_n_layer)); + } + hparams.audio_chunk_len = 0; + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + // one full window + using nt = clip_graph_nemotron_transcribe; + hparams.warmup_audio_size = (nt::window_left + nt::window_core + nt::window_right) * hparams.subsampling_factor; + } break; case PROJECTOR_TYPE_IDEFICS3: { // use default llava-uhd preprocessing params @@ -4073,6 +4098,87 @@ struct clip_model_loader { model.mm_model_mlp_2_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 1, "weight")); model.mm_model_mlp_3_w = get_tensor(string_format(TN_MVLM_PROJ_MLP, 3, "weight")); } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + hparams.mel_filters = get_vector(TN_MEL_FILTERS); + hparams.window = get_vector(TN_WINDOW); + + // ASR expert: blocks, pre_ln and post_ln come from the generic loader + model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight")); + for (int il = 0; il < hparams.n_layer; ++il) { + auto & layer = model.layers[il]; + layer.q_norm_b = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "bias")); + layer.k_norm_b = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "bias")); + } + + // speaker expert + const char * sp = "a.spk"; + model.spk_inp_proj_w = get_tensor(string_format(TN_A_SPK_INP_PROJ, "weight")); + model.spk_pre_ln_w = get_tensor(string_format(TN_LN_PRE, sp, "weight")); + model.spk_pre_ln_b = get_tensor(string_format(TN_LN_PRE, sp, "bias")); + model.spk_post_ln_w = get_tensor(string_format(TN_LN_POST, sp, "weight")); + model.spk_post_ln_b = get_tensor(string_format(TN_LN_POST, sp, "bias")); + model.spk_layers.resize(hparams.spk_n_layer); + for (int il = 0; il < hparams.spk_n_layer; ++il) { + auto & layer = model.spk_layers[il]; + layer.ln_1_w = get_tensor(string_format(TN_LN_1, sp, il, "weight")); + layer.ln_1_b = get_tensor(string_format(TN_LN_1, sp, il, "bias")); + layer.qkv_w = get_tensor(string_format(TN_ATTN_QKV, sp, il, "weight")); + layer.o_w = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "weight")); + layer.o_b = get_tensor(string_format(TN_ATTN_OUTPUT, sp, il, "bias")); + layer.ln_2_w = get_tensor(string_format(TN_LN_2, sp, il, "weight")); + layer.ln_2_b = get_tensor(string_format(TN_LN_2, sp, il, "bias")); + layer.ff_up_w = get_tensor(string_format(TN_FFN_UP, sp, il, "weight")); + layer.ff_up_b = get_tensor(string_format(TN_FFN_UP, sp, il, "bias")); + layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, sp, il, "weight")); + layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, sp, il, "bias")); + } + + // Sortformer head, fusion + model.spk_head_proj_w = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "weight")); + model.spk_head_proj_b = get_tensor(string_format(TN_A_SPK_HEAD, "proj", "bias")); + model.spk_head_up_w = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "weight")); + model.spk_head_up_b = get_tensor(string_format(TN_A_SPK_HEAD, "upsample", "bias")); + model.spk_head_fc1_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "weight")); + model.spk_head_fc1_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc1", "bias")); + model.spk_head_fc2_w = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "weight")); + model.spk_head_fc2_b = get_tensor(string_format(TN_A_SPK_HEAD, "fc2", "bias")); + model.fusion_asr_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "weight")); + model.fusion_asr_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "asr_norm", "bias")); + model.fusion_spk_norm_w = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "weight")); + model.fusion_spk_norm_b = get_tensor(string_format(TN_A_FUSION_NORM, "spk_norm", "bias")); + model.fusion_spk_kernel = get_tensor(TN_A_FUSION_KERNEL); + + model.spk_sil_emb = get_vector(TN_A_SPK_SIL_EMB); + + // the head layouts and the widths the graph and the speaker state use must match the tensors + hparams.spk_n_embd = model.spk_inp_proj_w->ne[1]; + const int64_t d_head = hparams.n_head > 0 ? hparams.n_embd / hparams.n_head : 0; + const int64_t d_head_spk = hparams.spk_n_head > 0 ? hparams.spk_n_embd / hparams.spk_n_head : 0; + const int64_t n_spk = model.spk_head_fc2_w->ne[1]; + // the graph splits each fused QKV into three views of the layer width + bool qkv_ok = true; + for (const auto & layer : model.layers) { + qkv_ok = qkv_ok && layer.qkv_w && layer.qkv_w->ne[0] == hparams.n_embd && layer.qkv_w->ne[1] == 3 * hparams.n_embd; + } + for (const auto & layer : model.spk_layers) { + qkv_ok = qkv_ok && layer.qkv_w->ne[0] == hparams.spk_n_embd && layer.qkv_w->ne[1] == 3 * hparams.spk_n_embd; + } + if (!qkv_ok || model.inp_proj_w->ne[1] != hparams.n_embd || d_head <= 0 || d_head * hparams.n_head != hparams.n_embd || + (hparams.n_embd_head > 0 && hparams.n_embd_head != d_head) || + hparams.audio_rope_dims <= 0 || hparams.audio_rope_dims % 2 != 0 || hparams.audio_rope_dims > d_head || + d_head_spk <= 0 || d_head_spk % 2 != 0 || d_head_spk * hparams.spk_n_head != hparams.spk_n_embd || + (int64_t) model.spk_sil_emb.size() != hparams.spk_n_embd || n_spk <= 0 || + model.fusion_spk_kernel->ne[0] != n_spk || model.fusion_spk_kernel->ne[1] != hparams.n_embd || + hparams.mel_filters.size() != (size_t) hparams.n_mel_bins * (hparams.audio_n_fft / 2 + 1) || + hparams.window.size() != (size_t) hparams.audio_window_len) { + throw std::runtime_error(string_format("%s: hparams do not match the tensor shapes\n", __func__)); + } + + // projection to the text model + model.mm_fc_w = get_tensor(string_format(TN_MM_AUDIO_FC, "weight")); + model.mm_fc_b = get_tensor(string_format(TN_MM_AUDIO_FC, "bias")); + } break; case PROJECTOR_TYPE_GRANITE_SPEECH: { model.inp_proj_w = get_tensor(string_format(TN_INP_PROJ, "weight")); @@ -5013,6 +5119,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { } n_patches = n; } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: case PROJECTOR_TYPE_PARAKEET: { n_patches = (img->nx() + (params.subsampling_factor - 1)) / params.subsampling_factor; @@ -5100,6 +5207,9 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32 params.n_threads = n_threads; params.out_embd = &out_batch_embd; + if (ctx->proj_type() == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + return clip_graph_nemotron_transcribe::encode_windows(ctx, ctx->model, ¶ms); + } return clip_encode(ctx, ¶ms); } @@ -6593,6 +6703,20 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size); } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + GGML_ASSERT(imgs.entries.size() == 1); + std::vector positions(ggml_nelements(get_inp_tensor("positions"))); + for (size_t i = 0; i < positions.size(); i++) { + positions[i] = (int32_t) i; + } + set_input_i32("positions", positions); + GGML_ASSERT(params->enc_state != nullptr); + set_input_f32("inp_norm", params->enc_state->mel_norm); + if (ggml_graph_get_tensor(gf, "spk_prev") != nullptr) { + set_input_f32("spk_prev", params->enc_state->spk_prev); + } + } break; case PROJECTOR_TYPE_PARAKEET: { GGML_ASSERT(imgs.entries.size() == 1); @@ -6899,6 +7023,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { GGML_ASSERT(t != nullptr); copy_gen_state_out(t, slots[i++]); } + } else if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE) { + clip_enc_state * st = params->enc_state; + ggml_tensor * chunk = ggml_graph_get_tensor(gf, "spk_chunk"); + ggml_tensor * preds = ggml_graph_get_tensor(gf, "spk_preds"); + GGML_ASSERT(chunk != nullptr && preds != nullptr); + st->spk_chunk.resize(ggml_nelements(chunk)); + st->spk_preds.resize(ggml_nelements(preds)); + ggml_backend_tensor_get(chunk, st->spk_chunk.data(), 0, ggml_nbytes(chunk)); + ggml_backend_tensor_get(preds, st->spk_preds.data(), 0, ggml_nbytes(preds)); } else if (params->enc_state != nullptr) { clip_enc_state * st = params->enc_state; ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail"); @@ -7057,6 +7190,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { case PROJECTOR_TYPE_LLAMA4: return ctx->model.mm_model_proj->ne[1]; case PROJECTOR_TYPE_QWEN2A: + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: return ctx->model.mm_fc_w->ne[1]; case PROJECTOR_TYPE_QWEN3A: return ctx->model.mm_2_w->ne[1]; diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index c75004d43725..dc816fa87c34 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -30,6 +30,22 @@ struct clip_graph_gemma4uv : clip_graph { ggml_cgraph * build() override; }; +struct clip_graph_nemotron_transcribe : clip_graph { + const clip_enc_state * state; // nullptr during warmup + clip_graph_nemotron_transcribe(clip_ctx * ctx, const clip_image_f32 & img, const clip_enc_state * state) + : clip_graph(ctx, img), state(state) {} + ggml_cgraph * build() override; + ggml_tensor * build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos); + + // one recording: encodes its windows in turn, carrying the speaker state between them + static bool encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params); + + // NeMo ParallelExpertEncoder online defaults, in output frames: 40 s windows with 4 s of context on each side + static constexpr int window_core = 500; + static constexpr int window_left = 50; + static constexpr int window_right = 50; +}; + struct clip_graph_pixtral : clip_graph { clip_graph_pixtral(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/models/nemotron-transcribe.cpp b/tools/mtmd/models/nemotron-transcribe.cpp new file mode 100644 index 000000000000..2ce20c81fc91 --- /dev/null +++ b/tools/mtmd/models/nemotron-transcribe.cpp @@ -0,0 +1,343 @@ +#include "models.h" + +#include +#include +#include +#include + +// NeMo ParallelExpertEncoder: ASR and speaker (Sortformer) transformers, speaker activity fused into the ASR states + +// a speaker is active in an 80 ms frame where its prediction exceeds this (speaker_activity_threshold) +static constexpr float SPK_ACTIVITY_THR = 0.5f; + +// NeMo streaming Sortformer: PEE online defaults and the checkpoint's sortformer_modules config +static constexpr int SPK_FIFO_LEN = 40; +static constexpr int SPK_CACHE_LEN = 188; +static constexpr int SPK_SIL_FRAMES = 1; +static constexpr float SPK_PRED_SCORE_THR = 0.25f; +static constexpr float SPK_BOOST_LATEST = 0.05f; +static constexpr float SPK_STRONG_BOOST_RATE = 0.75f; +static constexpr float SPK_WEAK_BOOST_RATE = 1.5f; +static constexpr float SPK_MIN_POS_RATE = 0.5f; + +// _compress_spkcache runs here on exp(score): its additive log terms become factors and -inf becomes 0 +static constexpr float SPK_SCORE_OFFSET = 2.0f; // -log(0.5) in _get_log_pred_scores +static constexpr float SPK_STRONG_BOOST = 4.0f; // _boost_topk_scores, scale_factor 2 +static constexpr float SPK_WEAK_BOOST = 2.0f; // _boost_topk_scores, scale_factor 1 +static constexpr float SPK_SPEECH_THR = 0.5f; // _disable_low_scores: a speaker's frames at or below it are not cached + +// the speaker expert has its own layers, width and head count, which build_vit does not take; full rotary +ggml_tensor * clip_graph_nemotron_transcribe::build_spk_encoder(ggml_tensor * cur, ggml_tensor * pos) { + const int n_head = hparams.spk_n_head; + const int64_t n_embd = cur->ne[0]; + const int64_t n_pos = cur->ne[1]; + const int64_t d_head = n_embd / n_head; + + cur = build_norm(cur, model.spk_pre_ln_w, model.spk_pre_ln_b, NORM_TYPE_NORMAL, eps, -1); + for (int il = 0; il < (int) model.spk_layers.size(); il++) { + const auto & layer = model.spk_layers[il]; + ggml_tensor * x = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il); + + ggml_tensor * qkv = build_mm(layer.qkv_w, x); + const size_t es = ggml_element_size(qkv); + ggml_tensor * q = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 0); + ggml_tensor * k = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], n_embd * es); + ggml_tensor * v = ggml_view_3d(ctx0, qkv, d_head, n_head, n_pos, d_head * es, qkv->nb[1], 2 * n_embd * es); + q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + + x = build_attn(layer.o_w, layer.o_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), il); + cur = ggml_add(ctx0, cur, x); + + x = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il); + x = build_ffn(x, layer.ff_up_w, layer.ff_up_b, nullptr, nullptr, layer.ff_down_w, layer.ff_down_b, FFN_GELU_ERF, il); + cur = ggml_add(ctx0, cur, x); + } + + return build_norm(cur, model.spk_post_ln_w, model.spk_post_ln_b, NORM_TYPE_NORMAL, eps, -1); +} + +ggml_cgraph * clip_graph_nemotron_transcribe::build() { + const int sub = hparams.subsampling_factor; + const int64_t n_mel = hparams.n_mel_bins; + const int64_t n_mel_frames = img.nx(); + const int64_t n_pos = (n_mel_frames + sub - 1) / sub; + GGML_ASSERT(img.ny() == n_mel); + + // warmup: the largest graph, with a full speaker cache and FIFO + const int64_t n_prev = state == nullptr ? SPK_CACHE_LEN + SPK_FIFO_LEN + : (int64_t) state->spk_prev.size() / hparams.spk_n_embd; + + // [n_frames, n_mel] log-mel: raw for the speaker expert, normalized over the recording (encode_windows) for the ASR one + ggml_tensor * inp = build_inp_raw(1); + ggml_tensor * inp_norm = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_mel_frames, n_mel); + ggml_set_name(inp_norm, "inp_norm"); + ggml_set_input(inp_norm); + auto stack_mel = [&](ggml_tensor * mel) { + return build_stack(ggml_cont(ctx0, ggml_transpose(ctx0, mel)), sub, n_mel); + }; + + // positions over [cache | FIFO | window]; the ASR expert takes the first n_pos + ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_prev + n_pos); + ggml_set_name(pos, "positions"); + ggml_set_input(pos); + + ggml_tensor * spk_prev = nullptr; + if (n_prev > 0) { + spk_prev = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.spk_n_embd, n_prev); + ggml_set_name(spk_prev, "spk_prev"); + ggml_set_input(spk_prev); + } + + ggml_tensor * asr_pos = ggml_view_1d(ctx0, pos, n_pos, 0); + auto add_pos = [&](ggml_tensor * cur, const clip_layer &) { + return ggml_rope_ext(ctx0, cur, asr_pos, nullptr, hparams.audio_rope_dims, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, + 0.0f, 1.0f, 0.0f, 0.0f); + }; + ggml_tensor * asr = build_vit(build_mm(model.inp_proj_w, stack_mel(inp_norm)), n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr, + add_pos); + cb(asr, "asr_out", -1); + + // speaker expert over [cache | FIFO | window]; the window's rows and predictions feed the next window's state + ggml_tensor * chunk = build_mm(model.spk_inp_proj_w, stack_mel(inp)); + ggml_set_name(chunk, "spk_chunk"); + ggml_set_output(chunk); + ggml_tensor * spk = spk_prev ? ggml_concat(ctx0, spk_prev, chunk, 1) : chunk; + const int64_t n_spk_pos = n_prev + n_pos; + spk = build_spk_encoder(spk, pos); + cb(spk, "spk_out", -1); + + // Sortformer head, 10 ms frames + ggml_tensor * cur = ggml_add(ctx0, build_mm(model.spk_head_proj_w, spk), model.spk_head_proj_b); + const int64_t d_tf = cur->ne[0]; + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_conv_1d(ctx0, model.spk_head_up_w, cur, 1, 1, 1); // [n_spk_pos, sub*d_tf] + cur = ggml_add(ctx0, cur, ggml_reshape_2d(ctx0, model.spk_head_up_b, 1, sub * d_tf)); + cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur)); + cur = ggml_reshape_2d(ctx0, cur, d_tf, sub * n_spk_pos); + cur = ggml_add(ctx0, build_mm(model.spk_head_fc1_w, ggml_relu(ctx0, cur)), model.spk_head_fc1_b); + cur = ggml_add(ctx0, build_mm(model.spk_head_fc2_w, ggml_relu(ctx0, cur)), model.spk_head_fc2_b); + ggml_tensor * probs = ggml_sigmoid(ctx0, cur); // [n_spk, sub*n_spk_pos] + cb(probs, "spk_probs", -1); + + // 80 ms predictions + ggml_tensor * preds = ggml_pool_2d(ctx0, probs, GGML_OP_POOL_AVG, 1, sub, 1, sub, 0, 0); // [n_spk, n_spk_pos] + ggml_set_name(preds, "spk_preds"); + ggml_set_output(preds); + + // fusion, on the window's rows + preds = ggml_view_2d(ctx0, preds, preds->ne[0], n_pos, preds->nb[1], n_prev * preds->nb[1]); + ggml_tensor * act = ggml_step(ctx0, ggml_scale_bias(ctx0, preds, 1.0f, -SPK_ACTIVITY_THR)); + act = build_norm(act, model.fusion_spk_norm_w, model.fusion_spk_norm_b, NORM_TYPE_NORMAL, eps, -1); + cur = build_norm(asr, model.fusion_asr_norm_w, model.fusion_asr_norm_b, NORM_TYPE_NORMAL, eps, -1); + cur = ggml_add(ctx0, cur, build_mm(model.fusion_spk_kernel, act)); + cb(cur, "fused", -1); + + cur = ggml_add(ctx0, build_mm(model.mm_fc_w, cur), model.mm_fc_b); + cb(cur, "projected", -1); + + ggml_build_forward_expand(gf, cur); + + return gf; +} + +// indices of the k largest of v[0, n), ties to the lower index +static std::vector spk_topk(const float * v, int n, int k) { + std::vector idx(n); + std::iota(idx.begin(), idx.end(), 0); + k = std::clamp(k, 0, n); + std::partial_sort(idx.begin(), idx.begin() + k, idx.end(), [&](int a, int b) { + return v[a] > v[b] || (v[a] == v[b] && a < b); + }); + idx.resize(k); + return idx; +} + +// NeMo SortformerModules._compress_spkcache: keep the SPK_CACHE_LEN highest-scoring rows, speaker by speaker +static void spk_compress_cache(std::vector & emb, std::vector & preds, int n, int d, int n_spk, + const std::vector & sil_emb) { + const int per_spk = std::max(0, SPK_CACHE_LEN / n_spk - SPK_SIL_FRAMES); + const int strong = (int) std::floor(per_spk * SPK_STRONG_BOOST_RATE); + const int weak = (int) std::floor(per_spk * SPK_WEAK_BOOST_RATE); + const int min_pos = (int) std::floor(per_spk * SPK_MIN_POS_RATE); + const float disabled = 0.0f; + const float latest = std::exp(SPK_BOOST_LATEST); + + // speaker-major: each speaker's n rows, then its silence rows (+inf) + const int nt = n + SPK_SIL_FRAMES; + std::vector scores((size_t) n_spk * nt, std::numeric_limits::infinity()); + for (int s = 0; s < n_spk; s++) { + float * sc = scores.data() + (size_t) s * nt; + int n_pos = 0; + for (int t = 0; t < n; t++) { + const float * pt = preds.data() + (size_t) t * n_spk; + float prod = 1.0f; + for (int j = 0; j < n_spk; j++) { + if (j != s) { + prod *= std::max(1.0f - pt[j], SPK_PRED_SCORE_THR); + } + } + prod *= std::max(pt[s], SPK_PRED_SCORE_THR); + sc[t] = pt[s] > SPK_SPEECH_THR ? prod * SPK_SCORE_OFFSET : disabled; + n_pos += sc[t] > 1.0f; + } + if (n_pos >= min_pos) { + for (int t = 0; t < n; t++) { + if (!(sc[t] > 1.0f)) { + sc[t] = disabled; + } + } + } + for (int t = SPK_CACHE_LEN; t < n; t++) { + sc[t] *= latest; + } + for (int t : spk_topk(sc, n, strong)) { + sc[t] *= SPK_STRONG_BOOST; + } + for (int t : spk_topk(sc, n, weak)) { + sc[t] *= SPK_WEAK_BOOST; + } + } + + std::vector pick = spk_topk(scores.data(), (int) scores.size(), SPK_CACHE_LEN); + const int placeholder = (int) scores.size(); + for (int & i : pick) { + if (scores[i] == disabled) { + i = placeholder; + } + } + std::sort(pick.begin(), pick.end()); + + std::vector out_emb((size_t) SPK_CACHE_LEN * d); + std::vector out_preds((size_t) SPK_CACHE_LEN * n_spk, 0.0f); + for (int i = 0; i < SPK_CACHE_LEN; i++) { + const int t = pick[i] == placeholder ? -1 : pick[i] % nt; + float * e = out_emb.data() + (size_t) i * d; + if (t < 0 || t >= n) { + std::copy(sil_emb.begin(), sil_emb.end(), e); + } else { + std::copy_n(emb.data() + (size_t) t * d, d, e); + std::copy_n(preds.data() + (size_t) t * n_spk, n_spk, out_preds.data() + (size_t) i * n_spk); + } + } + emb.swap(out_emb); + preds.swap(out_preds); +} + +// speaker cache, with its predictions, and FIFO of one recording +struct spk_stream { + std::vector cache, cache_preds; + std::vector fifo; +}; + +// every update has a full window, which overflows the FIFO and the cache (NeMo's update period of 300 never applies) +static_assert(clip_graph_nemotron_transcribe::window_core - SPK_FIFO_LEN > SPK_CACHE_LEN); + +// NeMo SortformerModules.streaming_update: core rows join the FIFO, its overflow moves to the cache, which is compressed +static void update_spk_state(spk_stream & ss, const clip_enc_state & st, const clip_model & model, int64_t core_begin, + int64_t n_core) { + const int64_t d = model.hparams.spk_n_embd; + const int64_t n_spk = model.spk_head_fc2_w->ne[1]; + const int64_t n_cache = (int64_t) ss.cache.size() / d; + const int64_t n_fifo = (int64_t) ss.fifo.size() / d; + const int64_t n_rows = (int64_t) st.spk_chunk.size() / d; + GGML_ASSERT((n_cache + n_fifo + n_rows) * n_spk == (int64_t) st.spk_preds.size() && core_begin + n_core <= n_rows); + const float * preds = st.spk_preds.data(); + const float * core_preds = preds + (n_cache + n_fifo + core_begin) * n_spk; + + std::vector fifo_preds(preds + n_cache * n_spk, preds + (n_cache + n_fifo) * n_spk); + ss.fifo.insert(ss.fifo.end(), st.spk_chunk.begin() + core_begin * d, st.spk_chunk.begin() + (core_begin + n_core) * d); + fifo_preds.insert(fifo_preds.end(), core_preds, core_preds + n_core * n_spk); + + const int64_t n_pop = n_fifo + n_core - SPK_FIFO_LEN; + ss.cache.insert(ss.cache.end(), ss.fifo.begin(), ss.fifo.begin() + n_pop * d); + ss.cache_preds.insert(ss.cache_preds.end(), fifo_preds.begin(), fifo_preds.begin() + n_pop * n_spk); + ss.fifo.erase(ss.fifo.begin(), ss.fifo.begin() + n_pop * d); + spk_compress_cache(ss.cache, ss.cache_preds, (int) (ss.cache.size() / d), (int) d, (int) n_spk, model.spk_sil_emb); +} + +// NeMo ParallelExpertEncoder._forward_online: windows with left/right context, one graph each, the speaker state carried +bool clip_graph_nemotron_transcribe::encode_windows(clip_ctx * ctx, const clip_model & model, clip_encode_params * params) { + GGML_ASSERT(params->imgs->entries.size() == 1 && params->out_embd != nullptr); + const clip_image_f32 & rec = params->imgs->entries[0]; + const auto & buf = rec.get_ro_buf(); + const int64_t n_frames = rec.nx(); + const int64_t n_rows = rec.ny(); + const int64_t n_embd = clip_n_mmproj_embd(ctx); + const int sub = model.hparams.subsampling_factor; + const int64_t core = (int64_t) window_core * sub; + const int64_t left = (int64_t) window_left * sub; + const int64_t right = (int64_t) window_right * sub; + const int64_t n_win = std::max(1, (n_frames + core - 1) / core); + GGML_ASSERT((int64_t) buf.size() == n_frames * n_rows); + + auto & out = *params->out_embd; + GGML_ASSERT(out.size() == (size_t) n_embd * clip_n_output_tokens(ctx, &rec)); + + // the ASR expert's input: per-feature normalization over the whole recording (NeMo normalize_batch, per_feature) + std::vector mean(n_rows), denom(n_rows); + for (int64_t r = 0; r < n_rows; r++) { + const float * row = buf.data() + r * n_frames; + double sum = 0.0; + double sq_diff_sum = 0.0; + for (int64_t t = 0; t < n_frames; t++) { + sum += (double) row[t]; + } + mean[r] = sum / n_frames; + for (int64_t t = 0; t < n_frames; t++) { + const double diff = (double) row[t] - mean[r]; + sq_diff_sum += diff * diff; + } + denom[r] = std::sqrt(sq_diff_sum / (n_frames - 1.0)) + 1e-5; + } + + spk_stream ss; + clip_enc_state st; + size_t n_out = 0; + for (int64_t w = 0; w < n_win; w++) { + const int64_t core_begin = w * core; + const int64_t core_end = std::min(core_begin + core, n_frames); + const int64_t win_begin = std::max(core_begin - left, 0); + const int64_t win_end = std::min(core_end + right, n_frames); + const int64_t win_len = win_end - win_begin; + + std::vector data((size_t) win_len * n_rows); + st.mel_norm.resize(data.size()); + for (int64_t r = 0; r < n_rows; r++) { + const float * src = buf.data() + r * n_frames + win_begin; + std::copy_n(src, win_len, data.data() + r * win_len); + for (int64_t t = 0; t < win_len; t++) { + st.mel_norm[r * win_len + t] = (float) ((src[t] - mean[r]) / denom[r]); + } + } + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.emplace_back(); + batch.entries[0].set_size({(int) win_len, (int) n_rows}, false, true); + batch.entries[0].cpy_buf(data); + + std::vector embd((size_t) n_embd * clip_n_output_tokens(ctx, &batch.entries[0])); + clip_encode_params wp = *params; + wp.imgs = &batch; + wp.out_embd = &embd; + wp.enc_state = &st; + if (!clip_encode(ctx, &wp)) { + return false; + } + + // output frames of the core, rounded up as NeMo's _asr_output_frame_boundary + const int64_t n_left = (core_begin - win_begin) / sub; + const int64_t n_core = (core_end - core_begin + sub - 1) / sub; + GGML_ASSERT((n_out + n_core) * n_embd <= out.size()); + std::copy_n(embd.data() + n_left * n_embd, n_core * n_embd, out.data() + n_out * n_embd); + n_out += n_core; + + if (w + 1 < n_win) { + update_spk_state(ss, st, model, n_left, n_core); + st.spk_prev = ss.cache; + st.spk_prev.insert(st.spk_prev.end(), ss.fifo.begin(), ss.fifo.end()); + } + } + GGML_ASSERT(n_out * n_embd == out.size()); + return true; +} diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 14b5a721357d..6e1ee23242d3 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -1385,13 +1385,8 @@ void mtmd_audio_preprocessor_parakeet::initialize() { cache.hann_window = hparams.window; } -bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, - size_t n_samples_in, - std::vector & output) const { - if (n_samples_in == 0) { - return false; - } - +// log-mel without normalization +void mtmd_audio_preprocessor_parakeet::log_mel(const float * samples, size_t n_samples_in, mtmd_audio_mel & out_full) const { filter_params params; params.n_mel = hparams.n_mel_bins; params.n_fft_bins = 1 + (hparams.audio_n_fft / 2); @@ -1422,7 +1417,6 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, std::vector samples_padded(n_samples_in + 2 * pad, 0.0f); std::copy(samples_preprocessed.begin(), samples_preprocessed.end(), samples_padded.begin() + pad); - mtmd_audio_mel out_full; out_full.n_mel = params.n_mel; out_full.n_len = (samples_padded.size() - frame_size) / frame_step + 1; out_full.n_len_org = out_full.n_len; @@ -1461,11 +1455,22 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, for (int iw = 0; iw < n_threads - 1; ++iw) { workers[iw].join(); } +} + +bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, + size_t n_samples_in, + std::vector & output) const { + if (n_samples_in == 0) { + return false; + } + + mtmd_audio_mel out_full; + log_mel(samples, n_samples_in, out_full); // Per-feature normalization (only on valid frames) { const double eps = 1e-5; - int valid_frames = n_samples_in / frame_step; + int valid_frames = n_samples_in / hparams.audio_hop_len; for (int j = 0; j < out_full.n_mel; j++) { double sum = 0.0; @@ -1497,6 +1502,32 @@ bool mtmd_audio_preprocessor_parakeet::preprocess(const float * samples, return true; } +// +// mtmd_audio_preprocessor_nemotron_transcribe implementation +// + +bool mtmd_audio_preprocessor_nemotron_transcribe::preprocess(const float * samples, + size_t n_samples, + std::vector & output) const { + if (n_samples < (size_t) hparams.audio_n_fft) { + return false; + } + + mtmd_audio_mel mel; + log_mel(samples, n_samples, mel); + + // frames past n_samples / hop are padding (NeMo get_seq_len); kept raw, encode_windows normalizes them for ASR + const int64_t n_frames = n_samples / hparams.audio_hop_len; + for (int64_t j = 1; j < mel.n_mel; j++) { + std::copy_n(mel.data.begin() + j * mel.n_len, n_frames, mel.data.begin() + j * n_frames); + } + mel.n_len = n_frames; + mel.n_len_org = n_frames; + mel.data.resize(mel.n_mel * n_frames); + output.push_back(std::move(mel)); + return true; +} + // mtmd_audio_preprocessor_gemma4ua // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index e9194ad5e339..087e4b8c0393 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -176,6 +176,9 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { void initialize() override; bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + protected: + void log_mel(const float * samples, size_t n_samples, mtmd_audio_mel & out) const; + private: mtmd_audio_cache cache; @@ -186,6 +189,12 @@ struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { const mtmd_audio_cache & cache, mtmd_audio_mel & mel); }; +// one chunk per recording: the raw log-mel of its valid frames +struct mtmd_audio_preprocessor_nemotron_transcribe : mtmd_audio_preprocessor_parakeet { + mtmd_audio_preprocessor_nemotron_transcribe(clip_ctx * ctx) : mtmd_audio_preprocessor_parakeet(ctx) { } + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; +}; + // // streaming ISTFT - converts spectrogram frames back to audio one frame at a time // diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index d4ea2385258c..c1b024a8840c 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -1153,6 +1153,10 @@ struct mtmd_context { { audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_NEMOTRON_TRANSCRIBE: + { + audio_preproc = std::make_unique(ctx_a); + } break; case PROJECTOR_TYPE_GEMMA4UA: { aud_beg = "<|audio>"; From d1044bc82712fbf08996080bef6298b24a27e9c7 Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Tue, 3 Nov 2026 15:36:44 +0100 Subject: [PATCH 026/286] model : enable NVFP4 LFM3 ggufs (#31608) --- src/models/lfm-base.cpp | 13 +++++++++---- src/models/lfm2.cpp | 6 +++--- src/models/lfm3.cpp | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/models/lfm-base.cpp b/src/models/lfm-base.cpp index 0dabf114bde1..74944c13c88b 100644 --- a/src/models/lfm-base.cpp +++ b/src/models/lfm-base.cpp @@ -117,9 +117,9 @@ ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(const llama_model & model, GGML_ASSERT(!model.layers[il].ffn_gate_b); GGML_ASSERT(!model.layers[il].ffn_down_b); return build_ffn(cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, + model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s, + model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s, + model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); } @@ -136,5 +136,10 @@ ggml_tensor * llm_build_lfm_base::build_lfm_moe_ffn(const llama_model & model, LLM_FFN_SILU, true, hparams.expert_weights_scale, static_cast(hparams.expert_gating_func), - il); + il, + nullptr, + nullptr, + model.layers[il].ffn_up_exps_s, + model.layers[il].ffn_gate_exps_s, + model.layers[il].ffn_down_exps_s); } diff --git a/src/models/lfm2.cpp b/src/models/lfm2.cpp index e18937faf3ca..f7d4f95b0aab 100644 --- a/src/models/lfm2.cpp +++ b/src/models/lfm2.cpp @@ -399,9 +399,9 @@ llama_model_lfm2::graph_decision::graph_decision(const llama_model & model, cons ggml_tensor * ffn_out = build_norm(cur, layer.ffn_norm, NULL, LLM_NORM_RMS, il); ffn_out = build_ffn(ffn_out, - layer.ffn_up, NULL, NULL, - layer.ffn_gate, NULL, NULL, - layer.ffn_down, NULL, NULL, + layer.ffn_up, NULL, layer.ffn_up_s, + layer.ffn_gate, NULL, layer.ffn_gate_s, + layer.ffn_down, NULL, layer.ffn_down_s, NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); cur = ggml_add(ctx0, cur, ffn_out); diff --git a/src/models/lfm3.cpp b/src/models/lfm3.cpp index b74dea41b0b0..20838f87fec3 100644 --- a/src/models/lfm3.cpp +++ b/src/models/lfm3.cpp @@ -129,7 +129,7 @@ llama_model_lfm3::graph::graph(const llama_model & model, const llm_gr q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il); cb(attn_out, "attn_out", il); - ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in); + ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in, model.layers[il].wqkv_gate_s); gate = ggml_sigmoid(ctx0, gate); cb(gate, "model.layers.{}.self_attn.gate", il); From 9309bf1bcfe8d52b0fb7ef906474db94041a5e0c Mon Sep 17 00:00:00 2001 From: Giulianno Date: Tue, 3 Nov 2026 08:24:45 -0700 Subject: [PATCH 027/286] model : Add support for Metis-1.6 models (#31134) Assisted-by: Claude --- conversion/__init__.py | 1 + conversion/base.py | 3 + conversion/metis.py | 204 +++++++++++++++++ convert_hf_to_gguf_update.py | 1 + gguf-py/gguf/constants.py | 125 ++++++++++ gguf-py/gguf/gguf_writer.py | 18 ++ gguf-py/gguf/tensor_mapping.py | 94 ++++++++ src/llama-arch.cpp | 68 ++++++ src/llama-arch.h | 37 +++ src/llama-context.cpp | 1 + src/llama-hparams.h | 8 + src/llama-model-saver.cpp | 32 +++ src/llama-model.cpp | 6 +- src/llama-model.h | 41 ++++ src/llama-quant.cpp | 10 + src/models/mamba-base.cpp | 6 +- src/models/metis16.cpp | 401 +++++++++++++++++++++++++++++++++ src/models/models.h | 33 +++ tests/fusion/MTL.csv | 2 + tests/test-llama-archs.cpp | 23 ++ 20 files changed, 1112 insertions(+), 2 deletions(-) create mode 100644 conversion/metis.py create mode 100644 src/models/metis16.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index dcbb97c1f85f..6cea1ec7890c 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -197,6 +197,7 @@ "MambaLMHeadModel": "mamba", "MapleForCausalLM": "maple", "MellumForCausalLM": "mellum", + "Metis16ForCausalLM": "metis", "MiMoV2FlashForCausalLM": "mimo", "MiMoV2ForCausalLM": "mimo", "MiniCPM3ForCausalLM": "minicpm", diff --git a/conversion/base.py b/conversion/base.py index fd880bbaba86..9d9d6e875adf 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -2116,6 +2116,9 @@ def get_vocab_base_pre(self, tokenizer) -> str: if chkhsh == "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4": # ref: https://huggingface.co/evilfreelancer/ruGPT3XL res = "gpt-2" + if chkhsh == "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9": + # ref: https://huggingface.co/Lernex/Metis-1.6-Logos + res = "gpt-2" if chkhsh == "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7": # ref: https://huggingface.co/LiquidAI/LFM2.5-8B-A1B res = "lfm2" diff --git a/conversion/metis.py b/conversion/metis.py new file mode 100644 index 000000000000..a073ed4df0a9 --- /dev/null +++ b/conversion/metis.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import math +from typing import Callable, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import LazyTorchTensor, ModelBase, TextModel, gguf + + +@ModelBase.register("Metis16ForCausalLM") +class Metis16Model(TextModel): + model_arch = gguf.MODEL_ARCH.METIS16 + + # read by generate_extra_tensors() or not used at inference: dropped in modify_tensors() + _folded = { + "pass_embeddings", "lm_head.weight", "depth_budget.multiplier", + "depth_memory.pass_embeddings", "depth_memory.anchor_embeddings", + "depth_memory.state_write.bias", "depth_memory.metadata_write.bias", + } + _folded_layer = { + "mixer_connection.controller.weight", "mixer_connection.controller.bias", "mixer_connection.mix_logits", + "mixer_connection.read_logits", "mixer_connection.write_logits", + "moe_connection.controller.weight", "moe_connection.controller.bias", "moe_connection.mix_logits", + "moe_connection.read_logits", "moe_connection.write_logits", + "moe.k_router.weight", "moe.k_router.bias", "moe.k_budget.multiplier", + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + hp = self.hparams + self.n_pass = hp["num_passes"] + self.attn_layers = list(hp["attention_layer_indices"]) + self._generated: set[str] = set() + # one cache slot per pass and layer, while the tensors exist once per physical layer + self.block_count = self.n_pass * hp["num_hidden_layers"] + + def set_vocab(self): + self._set_vocab_gpt2() + # turns end on the first eos id, so llama.cpp must stop on it too + eos = self.hparams.get("eos_token_id") + if isinstance(eos, list) and eos: + self.gguf_writer.add_eot_token_id(eos[0]) + + def set_gguf_parameters(self): + hp = self.hparams + super().set_gguf_parameters() + + n_layer = hp["num_hidden_layers"] + self.gguf_writer.add_head_count_kv([hp["num_key_value_heads"] if il % n_layer in self.attn_layers else 0 for il in range(self.block_count)]) + + self.gguf_writer.add_expert_feed_forward_length(hp["expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_feed_forward_length(hp["shared_expert_intermediate_size"]) + self.gguf_writer.add_expert_shared_count(1) + self.gguf_writer.add_moe_latent_size(hp["latent_dim"]) + + d_inner = hp["mamba_expand"] * hp["hidden_size"] + self.gguf_writer.add_ssm_conv_kernel(hp["mamba_d_conv"]) + self.gguf_writer.add_ssm_inner_size(d_inner) + self.gguf_writer.add_ssm_state_size(hp["mamba_d_state"]) + self.gguf_writer.add_ssm_time_step_rank(d_inner // hp["mamba_head_dim"]) + self.gguf_writer.add_ssm_group_count(hp["mamba_n_groups"]) + self.gguf_writer.add_ssm_norm_epsilon(hp["mamba_norm_eps"]) + + self.gguf_writer.add_hyper_connection_count(hp["n_streams"]) + self.gguf_writer.add_metis_pass_count(self.n_pass) + self.gguf_writer.add_metis_memory_length(hp["memory_dim"]) + self.gguf_writer.add_metis_route_length(hp["route_feature_dim"]) + self.gguf_writer.add_metis_continuation_logit_limit(hp["continuation_logit_limit"]) + self.gguf_writer.add_attention_pass_lora_rank(hp["attention_pass_lora_rank"]) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + # the checkpoint leaves these names suffix-less; add the suffix of the GGUF tensor so the mapper keeps it + if name.endswith((".mixer.impl.mixer.dt_bias", ".moe.selection_bias")): + name += ".bias" + elif (name.endswith((".moe.expert_embeddings", ".mixer.pass_gates")) + or name in ("depth_memory.stream_gate", "stream_embeddings")): + name += ".weight" + elif name == "depth_memory.stream_gate_bias": + name = "depth_memory.stream_gate.bias" + return super().filter_tensors((name, gen)) + + def _load(self, name: str) -> Tensor: + return LazyTorchTensor.to_eager(self.model_tensors[name]()) + + def _emit(self, name: str, data: Tensor) -> tuple[str, Tensor]: + self._generated.add(name) + return name, data + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + hp = self.hparams + r = hp["route_feature_dim"] + n_stream = hp["n_streams"] + T = gguf.MODEL_TENSOR + passes = self._load("pass_embeddings")[: self.n_pass] + + # the mixes depend only on the weights and the pass, so Sinkhorn runs once here + # the controller runs in the checkpoint dtype, as the reference does + for il in range(hp["num_hidden_layers"]): + for src, (mix, read, write) in ( + ("mixer_connection", (T.HC_ATTN_MIX, T.HC_ATTN_READ, T.HC_ATTN_WRITE)), + ("moe_connection", (T.HC_FFN_MIX, T.HC_FFN_READ, T.HC_FFN_WRITE)), + ): + prefix = f"layers.{il}.{src}." + ctrl_w, ctrl_b = self._load(prefix + "controller.weight"), self._load(prefix + "controller.bias") + mixes, reads, writes = [], [], [] + for p in range(self.n_pass): + delta = torch.nn.functional.linear(passes[p].to(ctrl_w.dtype), ctrl_w, ctrl_b).float() + m_delta, r_delta, w_delta = torch.split(delta, [n_stream * n_stream, n_stream, n_stream]) + log_m = self._load(prefix + "mix_logits").float() + m_delta.view(n_stream, n_stream) + for _ in range(hp["mhc_sinkhorn_iterations"]): + log_m = log_m - torch.logsumexp(log_m, dim=-1, keepdim=True) + log_m = log_m - torch.logsumexp(log_m, dim=-2, keepdim=True) + mixes.append(log_m.exp()) + reads.append(torch.softmax(self._load(prefix + "read_logits").float() + r_delta, dim=-1)) + writes.append(torch.softmax(self._load(prefix + "write_logits").float() + w_delta, dim=-1)) + act = ctrl_w.dtype + yield self._emit(self.format_tensor_name(mix, il), torch.stack(mixes).to(act).float()) + yield self._emit(self.format_tensor_name(read, il), torch.stack(reads).to(act).float()) + yield self._emit(self.format_tensor_name(write, il), torch.stack(writes).to(act).float()) + + # one bias per memory entry: write biases, pass and anchor embeddings, and the constant routed width + meta_w = self._load("depth_memory.metadata_write.weight").float() + mean_k = hp["num_experts_per_tok"] / hp["max_routed_k"] + base = (self._load("depth_memory.state_write.bias").float() + self._load("depth_memory.metadata_write.bias").float() + + mean_k * meta_w[:, r]) + pass_e = self._load("depth_memory.pass_embeddings").float() + anchor_e = self._load("depth_memory.anchor_embeddings").float() + n_anchor = len(self.attn_layers) + 1 + entry_bias = torch.stack([base + pass_e[p] + anchor_e[a] for p in range(self.n_pass) for a in range(n_anchor)]) + yield self._emit(self.format_tensor_name(T.DEPTH_MEM_ENTRY_BIAS), entry_bias) + + def _split_features(self, data_torch: Tensor, template: str, parts: tuple[str, ...], d: int, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # the graph reads the concatenated features one block at a time; each block is its own tensor + for i, part in enumerate(parts): + piece = data_torch[:, i * d:(i + 1) * d] if i < 3 else data_torch[:, 3 * d:] + yield from super().modify_tensors(piece, template.format(part), bid) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name in self._generated: + yield name, data_torch + return + if name in self._folded: + return + hp = self.hparams + d = hp["hidden_size"] + + if bid is None: + if name == "stream_embeddings.weight": + # the graph keeps the streams innermost: [n_stream, n_embd, n_tokens] + yield from super().modify_tensors(data_torch.T.contiguous(), name, bid) + return + if name == "depth_memory.metadata_write.weight": + # metadata input becomes [summary, entropy, confidence, continuation]; entropy arrives unnormalized + r = hp["route_feature_dim"] + meta_w = data_torch.float() + entropy_col = meta_w[:, r + 1:r + 2] / math.log(hp["n_routed_experts"]) + data_torch = torch.cat((meta_w[:, :r], entropy_col, meta_w[:, r + 2:]), dim=1) + elif name == "depth_memory.stream_gate.weight": + data_torch = data_torch.float() / math.sqrt(d) + elif name == "depth_memory.route_projection.weight": + yield from self._split_features(data_torch, "depth_memory.route_{}.weight", ("state", "retr", "diff", "hist"), d, bid) + return + elif name == "continuation.hidden.weight": + yield from self._split_features(data_torch, "continuation.hidden_{}.weight", ("state", "retr", "diff", "route"), d, bid) + return + yield from super().modify_tensors(data_torch, name, bid) + return + + local = name.removeprefix(f"layers.{bid}.") + if local in self._folded_layer: + return + + if local == "mixer.impl.mixer.A_log": + data_torch = -torch.exp(data_torch).unsqueeze(-1) + elif local == "mixer.impl.mixer.D": + data_torch = data_torch.unsqueeze(-1) + elif local == "mixer.impl.mixer.conv1d.weight": + data_torch = data_torch.squeeze(1) + elif local == "mixer.impl.mixer.norm.weight": + data_torch = data_torch.reshape(hp["mamba_n_groups"], -1) + elif local == "mixer.pass_gates.weight": + # the reference applies tanh(gate) in the checkpoint dtype + data_torch = torch.tanh(data_torch[: self.n_pass]).to(torch.bfloat16).float() + elif local == "moe.shared_expert.gate_up.weight": + gate, up = data_torch.chunk(2, dim=0) + yield from super().modify_tensors(gate, name.replace("gate_up", "gate"), bid) + yield from super().modify_tensors(up, name.replace("gate_up", "up"), bid) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + # FP32 in the checkpoint, tiny and read as rows, or computed here at full precision: keep FP32 + if (new_name.startswith(("cont_", "depth_mem_entry_bias", "depth_mem_stream_gate")) + or new_name.endswith(("ffn_exp_embd.weight", "ffn_gate_inp.weight", "hc_attn_mix.weight", "hc_attn_read.weight", + "hc_attn_write.weight", "hc_ffn_mix.weight", "hc_ffn_read.weight", "hc_ffn_write.weight"))): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index 37df033ef3d6..3b8a33597787 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -198,6 +198,7 @@ class TOKENIZER_TYPE(IntEnum): # jina-v2-de variants {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/aari1995/German_Semantic_V3", "chkhsh": "b3d1dd861f1d4c5c0d2569ce36baf3f90fe8a102db3de50dd71ff860d91be3df"}, {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/evilfreelancer/ruGPT3XL", "chkhsh": "0fe1cf6eda062318a1af7270f3331a85c539a01778ff948e24388e949c5282f4"}, + {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Lernex/Metis-1.6-Logos", "chkhsh": "df1bc34a6366466bcb36a9a1b1cc0afdb08cfe21c9250602c930f159085390a9"}, # lfm2 variants {"name": "lfm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LiquidAI/LFM2.5-8B-A1B", "chkhsh": "9e454714343b69b99b71795c1d27a68c2a1d15dab111f4d353109f966af29da7"}, # hrm-text (DFM Mimir) is SPM-style BPE: normalizer maps ' ' -> '▁', merges diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index ad4e675f69a8..c4c4a4c65eda 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -203,6 +203,7 @@ class Attention: ICLR_LORA_RANK = "{arch}.attention.iclr_lora_rank" VALUE_RESIDUAL_MIX_LORA_RANK = "{arch}.attention.value_residual_mix_lora_rank" GATE_LORA_RANK = "{arch}.attention.gate_lora_rank" + PASS_LORA_RANK = "{arch}.attention.pass_lora_rank" REL_BUCKETS_COUNT = "{arch}.attention.relative_buckets_count" SLIDING_WINDOW = "{arch}.attention.sliding_window" SCALE = "{arch}.attention.scale" @@ -315,6 +316,12 @@ class HRM: L_CYCLES = "{arch}.hrm.l_cycles" PREFIX_LM = "{arch}.hrm.prefix_lm" + class Metis: + PASS_COUNT = "{arch}.metis.pass_count" + MEMORY_LENGTH = "{arch}.metis.memory_length" + ROUTE_LENGTH = "{arch}.metis.route_length" + CONTINUATION_LOGIT_LIMIT = "{arch}.metis.continuation_logit_limit" + class SSM: CONV_KERNEL = "{arch}.ssm.conv_kernel" INNER_SIZE = "{arch}.ssm.inner_size" @@ -322,6 +329,7 @@ class SSM: TIME_STEP_RANK = "{arch}.ssm.time_step_rank" GROUP_COUNT = "{arch}.ssm.group_count" DT_B_C_RMS = "{arch}.ssm.dt_b_c_rms" + NORM_EPSILON = "{arch}.ssm.norm_epsilon" class KDA: HEAD_DIM = "{arch}.kda.head_dim" @@ -681,6 +689,7 @@ class MODEL_ARCH(IntEnum): NEMOTRON = auto() NEMOTRON_H = auto() NEMOTRON_H_MOE = auto() + METIS16 = auto() EXAONE = auto() EXAONE4 = auto() EXAONE_MOE = auto() @@ -830,6 +839,36 @@ class MODEL_TENSOR(IntEnum): FFN_GATE_TID2EID = auto() MOE_LATENT_DOWN = auto() # nemotron 3 super MOE_LATENT_UP = auto() # nemotron 3 super + STREAM_EMBD = auto() + DEPTH_MEM_STATE_WRITE = auto() + DEPTH_MEM_META_WRITE = auto() + DEPTH_MEM_ENTRY_BIAS = auto() + DEPTH_MEM_QUERY = auto() + DEPTH_MEM_KEY = auto() + DEPTH_MEM_VALUE = auto() + DEPTH_MEM_OUT = auto() + DEPTH_MEM_ROUTE_STATE = auto() + DEPTH_MEM_ROUTE_RETR = auto() + DEPTH_MEM_ROUTE_DIFF = auto() + DEPTH_MEM_ROUTE_HIST = auto() + DEPTH_MEM_ROUTE = auto() + DEPTH_MEM_STREAM_GATE = auto() + CONT_HIDDEN_STATE = auto() + CONT_HIDDEN_RETR = auto() + CONT_HIDDEN_DIFF = auto() + CONT_HIDDEN_ROUTE = auto() + CONT_HIDDEN = auto() + CONT_OUTPUT = auto() + HC_ATTN_MIX = auto() + HC_ATTN_READ = auto() + HC_ATTN_WRITE = auto() + HC_FFN_MIX = auto() + HC_FFN_READ = auto() + HC_FFN_WRITE = auto() + ATTN_PASS_LORA_DOWN = auto() + ATTN_PASS_LORA_UP = auto() + ATTN_PASS_GATE = auto() + FFN_EXP_EMBD = auto() FFN_RESID_ALPHA = auto() FFN_RESID_BETA = auto() FFN_PRE_EXP_NORM = auto() @@ -1558,6 +1597,7 @@ class MODEL_TENSOR(IntEnum): MODEL_ARCH.NEMOTRON: "nemotron", MODEL_ARCH.NEMOTRON_H: "nemotron_h", MODEL_ARCH.NEMOTRON_H_MOE: "nemotron_h_moe", + MODEL_ARCH.METIS16: "metis16", MODEL_ARCH.EXAONE: "exaone", MODEL_ARCH.EXAONE4: "exaone4", MODEL_ARCH.EXAONE_MOE: "exaone-moe", @@ -1708,6 +1748,36 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_GATE_TID2EID: "blk.{bid}.ffn_gate_tid2eid", MODEL_TENSOR.MOE_LATENT_DOWN: "blk.{bid}.ffn_latent_down", # nemotron 3 super MODEL_TENSOR.MOE_LATENT_UP: "blk.{bid}.ffn_latent_up", # nemotron 3 super + MODEL_TENSOR.STREAM_EMBD: "stream_embd", + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: "depth_mem_state_write", + MODEL_TENSOR.DEPTH_MEM_META_WRITE: "depth_mem_meta_write", + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS: "depth_mem_entry_bias", + MODEL_TENSOR.DEPTH_MEM_QUERY: "depth_mem_query", + MODEL_TENSOR.DEPTH_MEM_KEY: "depth_mem_key", + MODEL_TENSOR.DEPTH_MEM_VALUE: "depth_mem_value", + MODEL_TENSOR.DEPTH_MEM_OUT: "depth_mem_out", + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: "depth_mem_route_state", + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: "depth_mem_route_retr", + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: "depth_mem_route_diff", + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: "depth_mem_route_hist", + MODEL_TENSOR.DEPTH_MEM_ROUTE: "depth_mem_route", + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: "depth_mem_stream_gate", + MODEL_TENSOR.CONT_HIDDEN_STATE: "cont_hidden_state", + MODEL_TENSOR.CONT_HIDDEN_RETR: "cont_hidden_retr", + MODEL_TENSOR.CONT_HIDDEN_DIFF: "cont_hidden_diff", + MODEL_TENSOR.CONT_HIDDEN_ROUTE: "cont_hidden_route", + MODEL_TENSOR.CONT_HIDDEN: "cont_hidden", + MODEL_TENSOR.CONT_OUTPUT: "cont_output", + MODEL_TENSOR.HC_ATTN_MIX: "blk.{bid}.hc_attn_mix", + MODEL_TENSOR.HC_ATTN_READ: "blk.{bid}.hc_attn_read", + MODEL_TENSOR.HC_ATTN_WRITE: "blk.{bid}.hc_attn_write", + MODEL_TENSOR.HC_FFN_MIX: "blk.{bid}.hc_ffn_mix", + MODEL_TENSOR.HC_FFN_READ: "blk.{bid}.hc_ffn_read", + MODEL_TENSOR.HC_FFN_WRITE: "blk.{bid}.hc_ffn_write", + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: "blk.{bid}.attn_pass_lora_down", + MODEL_TENSOR.ATTN_PASS_LORA_UP: "blk.{bid}.attn_pass_lora_up", + MODEL_TENSOR.ATTN_PASS_GATE: "blk.{bid}.attn_pass_gate", + MODEL_TENSOR.FFN_EXP_EMBD: "blk.{bid}.ffn_exp_embd", MODEL_TENSOR.FFN_RESID_ALPHA: "blk.{bid}.ffn_resid_alpha", MODEL_TENSOR.FFN_RESID_BETA: "blk.{bid}.ffn_resid_beta", MODEL_TENSOR.FFN_PRE_EXP_NORM: "blk.{bid}.ffn_pre_exp_norm", @@ -4917,6 +4987,61 @@ class MODEL_TENSOR(IntEnum): MODEL_TENSOR.FFN_DOWN, MODEL_TENSOR.FFN_UP, ], + MODEL_ARCH.METIS16: [ + MODEL_TENSOR.TOKEN_EMBD, + MODEL_TENSOR.OUTPUT_NORM, + MODEL_TENSOR.OUTPUT, + MODEL_TENSOR.ATTN_NORM, + MODEL_TENSOR.FFN_NORM, + MODEL_TENSOR.STREAM_EMBD, + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE, + MODEL_TENSOR.DEPTH_MEM_META_WRITE, + MODEL_TENSOR.DEPTH_MEM_ENTRY_BIAS, + MODEL_TENSOR.DEPTH_MEM_QUERY, + MODEL_TENSOR.DEPTH_MEM_KEY, + MODEL_TENSOR.DEPTH_MEM_VALUE, + MODEL_TENSOR.DEPTH_MEM_OUT, + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE, + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR, + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF, + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST, + MODEL_TENSOR.DEPTH_MEM_ROUTE, + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE, + MODEL_TENSOR.CONT_HIDDEN_STATE, + MODEL_TENSOR.CONT_HIDDEN_RETR, + MODEL_TENSOR.CONT_HIDDEN_DIFF, + MODEL_TENSOR.CONT_HIDDEN_ROUTE, + MODEL_TENSOR.CONT_HIDDEN, + MODEL_TENSOR.CONT_OUTPUT, + MODEL_TENSOR.HC_ATTN_MIX, + MODEL_TENSOR.HC_ATTN_READ, + MODEL_TENSOR.HC_ATTN_WRITE, + MODEL_TENSOR.HC_FFN_MIX, + MODEL_TENSOR.HC_FFN_READ, + MODEL_TENSOR.HC_FFN_WRITE, + MODEL_TENSOR.ATTN_PASS_LORA_DOWN, + MODEL_TENSOR.ATTN_PASS_LORA_UP, + MODEL_TENSOR.ATTN_PASS_GATE, + MODEL_TENSOR.FFN_EXP_EMBD, + MODEL_TENSOR.SSM_IN, + MODEL_TENSOR.SSM_CONV1D, + MODEL_TENSOR.SSM_DT, + MODEL_TENSOR.SSM_A, + MODEL_TENSOR.SSM_D, + MODEL_TENSOR.SSM_NORM, + MODEL_TENSOR.SSM_OUT, + MODEL_TENSOR.ATTN_QKV, + MODEL_TENSOR.ATTN_OUT, + MODEL_TENSOR.MOE_LATENT_DOWN, + MODEL_TENSOR.MOE_LATENT_UP, + MODEL_TENSOR.FFN_GATE_INP, + MODEL_TENSOR.FFN_EXP_PROBS_B, + MODEL_TENSOR.FFN_GATE_UP_EXP, + MODEL_TENSOR.FFN_DOWN_EXP, + MODEL_TENSOR.FFN_GATE_SHEXP, + MODEL_TENSOR.FFN_UP_SHEXP, + MODEL_TENSOR.FFN_DOWN_SHEXP, + ], MODEL_ARCH.NEMOTRON_H: [ MODEL_TENSOR.TOKEN_EMBD, MODEL_TENSOR.OUTPUT_NORM, diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 56e8938c4f63..0e7aa5012c4b 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -975,6 +975,18 @@ def add_hrm_l_cycles(self, value: int) -> None: def add_hrm_prefix_lm(self, value: bool) -> None: self.add_bool(Keys.HRM.PREFIX_LM.format(arch=self.arch), value) + def add_metis_pass_count(self, value: int) -> None: + self.add_uint32(Keys.Metis.PASS_COUNT.format(arch=self.arch), value) + + def add_metis_memory_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.MEMORY_LENGTH.format(arch=self.arch), value) + + def add_metis_route_length(self, value: int) -> None: + self.add_uint32(Keys.Metis.ROUTE_LENGTH.format(arch=self.arch), value) + + def add_metis_continuation_logit_limit(self, value: float) -> None: + self.add_float32(Keys.Metis.CONTINUATION_LOGIT_LIMIT.format(arch=self.arch), value) + def add_adapter_count(self, count: int) -> None: self.add_uint32(Keys.Adapters.COUNT.format(arch=self.arch), count) @@ -1086,6 +1098,9 @@ def add_attention_output_group_count(self, count: int) -> None: def add_attention_output_lora_rank(self, length: int) -> None: self.add_uint32(Keys.Attention.OUTPUT_LORA_RANK.format(arch=self.arch), length) + def add_attention_pass_lora_rank(self, length: int) -> None: + self.add_uint32(Keys.Attention.PASS_LORA_RANK.format(arch=self.arch), length) + def add_attention_compress_ratios(self, values: Sequence[int]) -> None: self.add_array(Keys.Attention.COMPRESS_RATIOS.format(arch=self.arch), values) @@ -1267,6 +1282,9 @@ def add_ssm_group_count(self, value: int) -> None: def add_ssm_dt_b_c_rms(self, value: bool) -> None: self.add_bool(Keys.SSM.DT_B_C_RMS.format(arch=self.arch), value) + def add_ssm_norm_epsilon(self, value: float) -> None: + self.add_float32(Keys.SSM.NORM_EPSILON.format(arch=self.arch), value) + def add_expert_latent_length(self, value: int) -> None: self.add_uint32(Keys.LLM.EXPERT_LATENT_LENGTH.format(arch=self.arch), value) diff --git a/gguf-py/gguf/tensor_mapping.py b/gguf-py/gguf/tensor_mapping.py index 2602bfb61b3d..ff22a60bc6e4 100644 --- a/gguf-py/gguf/tensor_mapping.py +++ b/gguf-py/gguf/tensor_mapping.py @@ -35,6 +35,7 @@ class TensorNameMap: "model.transformer.wte", # llada "embed_tokens", # qwen3-embedding "model.embed", # talkie + "embedding", # metis16 ), # Masked embeddings @@ -224,6 +225,7 @@ class TensorNameMap: block_mappings_cfg: dict[MODEL_TENSOR, tuple[str, ...]] = { # Attention norm MODEL_TENSOR.ATTN_NORM: ( + "layers.{bid}.mixer_connection.norm", # metis16 "model.lilicorr.layers.{bid}.attn_norm", # lilicorr "gpt_neox.layers.{bid}.input_layernorm", # gptneox "transformer.h.{bid}.ln_1", # gpt2 gpt-j refact qwen jais exaone @@ -294,6 +296,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 "head.layers.{bid}.self_attn.in_proj", # laya "layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text + "layers.{bid}.mixer.qkv", # metis16 ), # Attention query @@ -401,6 +404,7 @@ class TensorNameMap: "model.layers.{bid}.self_attn.language_expert_dense", # cogvlm "model.blocks.{bid}.attn.attn_resid", # talkie "head.layers.{bid}.self_attn.out_proj", # laya + "layers.{bid}.mixer.out", # metis16 ), # Attention output norm @@ -449,6 +453,7 @@ class TensorNameMap: # Feed-forward norm MODEL_TENSOR.FFN_NORM: ( + "layers.{bid}.moe_connection.norm", # metis16 "model.lilicorr.layers.{bid}.mlp_norm", # lilicorr "model.lilicorr.feature_mlp.{bid}.norm", # lilicorr "gpt_neox.layers.{bid}.post_attention_layernorm", # gptneox @@ -536,6 +541,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.gate", # nemotron-h-moe "model.layers.{bid}.moe.gate", # step3.5 "model.layers.{bid}.router.proj", # gemma4 + "layers.{bid}.moe.expert_router", # metis16 ), MODEL_TENSOR.FFN_GATE_INP_SHEXP: ( @@ -543,6 +549,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_EXP_PROBS_B: ( + "layers.{bid}.moe.selection_bias", # metis16 "model.layers.{bid}.mlp.gate.e_score_correction", # deepseek-v3 dots1 "model.layers.{bid}.mlp.gate.e_score_correction_bias", # muse-spark "model.layers.{bid}.mlp.moe_statics.e_score_correction", # ernie4.5-moe @@ -617,6 +624,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_UP_SHEXP: ( + "layers.{bid}.moe.shared_expert.up", # metis16 "model.layers.{bid}.mlp.shared_expert.up_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.up_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.up_proj", # llama4 @@ -670,6 +678,7 @@ class TensorNameMap: ), MODEL_TENSOR.FFN_GATE_SHEXP: ( + "layers.{bid}.moe.shared_expert.gate", # metis16 "model.layers.{bid}.mlp.shared_expert.gate_proj", # qwen2moe "model.layers.{bid}.mlp.shared_experts.gate_proj", # deepseek deepseek2 "model.layers.{bid}.feed_forward.shared_expert.gate_proj", # llama4 @@ -686,18 +695,94 @@ class TensorNameMap: MODEL_TENSOR.FFN_GATE_UP_EXP: ( "model.layers.{bid}.mlp.experts.gate_up_proj", "model.layers.{bid}.experts.gate_up_proj", # gemma4 + "layers.{bid}.moe.local_experts.gate_up", # metis16 ), MODEL_TENSOR.MOE_LATENT_DOWN: ( "backbone.layers.{bid}.mixer.fc1_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.pre_expert_proj", # muse-spark + "layers.{bid}.moe.latent_down", # metis16 ), MODEL_TENSOR.MOE_LATENT_UP: ( "backbone.layers.{bid}.mixer.fc2_latent_proj", # nemotron 3 super "model.layers.{bid}.mlp.post_expert_proj", # muse-spark + "layers.{bid}.moe.latent_up", # metis16 ), + MODEL_TENSOR.STREAM_EMBD: ( + "stream_embeddings", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STATE_WRITE: ( + "depth_memory.state_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_META_WRITE: ( + "depth_memory.metadata_write", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_QUERY: ( + "depth_memory.query", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_KEY: ( + "depth_memory.key", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_VALUE: ( + "depth_memory.value", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_OUT: ( + "depth_memory.output", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_STATE: ( + "depth_memory.route_state", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_RETR: ( + "depth_memory.route_retr", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_DIFF: ( + "depth_memory.route_diff", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE_HIST: ( + "depth_memory.route_hist", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_ROUTE: ( + "depth_memory.route_projection", # metis16 + ), + MODEL_TENSOR.DEPTH_MEM_STREAM_GATE: ( + "depth_memory.stream_gate", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_STATE: ( + "continuation.hidden_state", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_RETR: ( + "continuation.hidden_retr", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_DIFF: ( + "continuation.hidden_diff", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN_ROUTE: ( + "continuation.hidden_route", # metis16 + ), + MODEL_TENSOR.CONT_HIDDEN: ( + "continuation.hidden", # metis16 + ), + MODEL_TENSOR.CONT_OUTPUT: ( + "continuation.output", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_DOWN: ( + "layers.{bid}.mixer.pass_lora_down", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_LORA_UP: ( + "layers.{bid}.mixer.pass_lora_up", # metis16 + ), + MODEL_TENSOR.ATTN_PASS_GATE: ( + "layers.{bid}.mixer.pass_gates", # metis16 + ), + MODEL_TENSOR.FFN_EXP_EMBD: ( + "layers.{bid}.moe.expert_embeddings", # metis16 + ), + + + + MODEL_TENSOR.FFN_PRE_EXP_NORM: ( "model.layers.{bid}.mlp.pre_expert_norm", # muse-spark ), @@ -759,6 +844,7 @@ class TensorNameMap: "model.layers.{bid}.block_sparse_moe.experts.down", # smallthinker "model.layers.{bid}.moe.down_proj", # step3.5 "model.layers.{bid}.experts.down_proj", # gemma4 + "layers.{bid}.moe.local_experts.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_SHEXP: ( @@ -771,6 +857,7 @@ class TensorNameMap: "backbone.layers.{bid}.mixer.shared_experts.down_proj", # nemotron-h-moe "model.layers.{bid}.block_sparse_moe.shared_experts.down_proj", # kimi "model.layers.{bid}.share_expert.down_proj", # step3.5 + "layers.{bid}.moe.shared_expert.down", # metis16 ), MODEL_TENSOR.FFN_DOWN_CHEXP: ( @@ -906,6 +993,7 @@ class TensorNameMap: "model.layers.{bid}.mamba.in_proj", # jamba falcon-h1 granite-hybrid "model.layers.layers.{bid}.mixer.in_proj", # plamo2 "model.layers.{bid}.linear_attn.in_proj_qkvz", # qwen3next + "layers.{bid}.mixer.impl.mixer.in_proj", # metis16 ), MODEL_TENSOR.SSM_CONV1D: ( @@ -915,6 +1003,7 @@ class TensorNameMap: "model.layers.layers.{bid}.mixer.conv1d", # plamo2 "model.layers.{bid}.linear_attn.conv1d", # qwen3next "layers.{bid}.linear_attn.conv1d", # qwen3.5 text + "layers.{bid}.mixer.impl.mixer.conv1d", # metis16 ), MODEL_TENSOR.SSM_X: ( @@ -925,6 +1014,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_DT: ( + "layers.{bid}.mixer.impl.mixer.dt_bias", # metis16 "model.layers.{bid}.dt_proj", # mamba-hf "backbone.layers.{bid}.mixer.dt_proj", # mamba "model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid @@ -942,6 +1032,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_A: ( + "layers.{bid}.mixer.impl.mixer.A_log", # metis16 "model.layers.{bid}.A_log", # mamba-hf "backbone.layers.{bid}.mixer.A_log", # mamba "model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid @@ -965,6 +1056,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_D: ( + "layers.{bid}.mixer.impl.mixer.D", # metis16 "model.layers.{bid}.D", # mamba-hf "backbone.layers.{bid}.mixer.D", # mamba "model.layers.{bid}.mamba.D", # jamba falcon-h1 granite-hybrid @@ -972,6 +1064,7 @@ class TensorNameMap: ), MODEL_TENSOR.SSM_NORM: ( + "layers.{bid}.mixer.impl.mixer.norm", # metis16 "model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid "model.layers.{bid}.linear_attn.norm", # qwen3next "layers.{bid}.linear_attn.norm", # qwen3.5 text @@ -987,6 +1080,7 @@ class TensorNameMap: "model.layers.{bid}.linear_attn.out_proj", # qwen3next "layers.{bid}.linear_attn.out_proj", # qwen3.5 text "model.layers.layers.{bid}.mixer.out_proj", # plamo2 + "layers.{bid}.mixer.impl.mixer.out_proj", # metis16 ), MODEL_TENSOR.SSM_ALPHA: ( diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 525f91031c0c..92a13e985875 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -102,6 +102,7 @@ static const std::map LLM_ARCH_NAMES = { { LLM_ARCH_NEMOTRON, "nemotron" }, { LLM_ARCH_NEMOTRON_H, "nemotron_h" }, { LLM_ARCH_NEMOTRON_H_MOE, "nemotron_h_moe" }, + { LLM_ARCH_METIS16, "metis16" }, { LLM_ARCH_EXAONE, "exaone" }, { LLM_ARCH_EXAONE4, "exaone4" }, { LLM_ARCH_EXAONE_MOE, "exaone-moe" }, @@ -272,6 +273,10 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_HRM_H_CYCLES, "%s.hrm.h_cycles" }, { LLM_KV_HRM_L_CYCLES, "%s.hrm.l_cycles" }, { LLM_KV_HRM_PREFIX_LM, "%s.hrm.prefix_lm" }, + { LLM_KV_METIS_PASS_COUNT, "%s.metis.pass_count" }, + { LLM_KV_METIS_MEMORY_LENGTH, "%s.metis.memory_length" }, + { LLM_KV_METIS_ROUTE_LENGTH, "%s.metis.route_length" }, + { LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, "%s.metis.continuation_logit_limit" }, { LLM_KV_MUDD_FEED_FORWARD_LENGTH, "%s.mudd.feed_forward_length" }, { LLM_KV_MUDD_TAP_COUNT, "%s.mudd.tap_count" }, { LLM_KV_MUDD_TAP_INDICES, "%s.mudd.tap_indices" }, @@ -329,6 +334,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, "%s.attention.kv_source_layers" }, { LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, "%s.attention.output_group_count" }, { LLM_KV_ATTENTION_OUTPUT_LORA_RANK, "%s.attention.output_lora_rank" }, + { LLM_KV_ATTENTION_PASS_LORA_RANK, "%s.attention.pass_lora_rank" }, { LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, "%s.attention.compress_rope_freq_base" }, { LLM_KV_ATTENTION_COMPRESS_RATIOS, "%s.attention.compress_ratios" }, { LLM_KV_ATTENTION_SHARED_KV_LAYERS, "%s.attention.shared_kv_layers" }, @@ -387,6 +393,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_SSM_TIME_STEP_RANK, "%s.ssm.time_step_rank" }, { LLM_KV_SSM_GROUP_COUNT, "%s.ssm.group_count" }, { LLM_KV_SSM_DT_B_C_RMS, "%s.ssm.dt_b_c_rms" }, + { LLM_KV_SSM_NORM_EPS, "%s.ssm.norm_epsilon" }, { LLM_KV_KDA_HEAD_DIM, "%s.kda.head_dim" }, { LLM_KV_KDA_SAFE_GATE, "%s.kda.safe_gate" }, @@ -548,6 +555,36 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_FFN_EXP_PROBS_B_VL, "blk.%d.exp_probs_b_vl" }, { LLM_TENSOR_FFN_LATENT_DOWN, "blk.%d.ffn_latent_down" }, { LLM_TENSOR_FFN_LATENT_UP, "blk.%d.ffn_latent_up" }, + { LLM_TENSOR_STREAM_EMBD, "stream_embd" }, + { LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "depth_mem_state_write" }, + { LLM_TENSOR_DEPTH_MEM_META_WRITE, "depth_mem_meta_write" }, + { LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "depth_mem_entry_bias" }, + { LLM_TENSOR_DEPTH_MEM_QUERY, "depth_mem_query" }, + { LLM_TENSOR_DEPTH_MEM_KEY, "depth_mem_key" }, + { LLM_TENSOR_DEPTH_MEM_VALUE, "depth_mem_value" }, + { LLM_TENSOR_DEPTH_MEM_OUT, "depth_mem_out" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "depth_mem_route_state" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "depth_mem_route_retr" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "depth_mem_route_diff" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "depth_mem_route_hist" }, + { LLM_TENSOR_DEPTH_MEM_ROUTE, "depth_mem_route" }, + { LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "depth_mem_stream_gate" }, + { LLM_TENSOR_CONT_HIDDEN_STATE, "cont_hidden_state" }, + { LLM_TENSOR_CONT_HIDDEN_RETR, "cont_hidden_retr" }, + { LLM_TENSOR_CONT_HIDDEN_DIFF, "cont_hidden_diff" }, + { LLM_TENSOR_CONT_HIDDEN_ROUTE, "cont_hidden_route" }, + { LLM_TENSOR_CONT_HIDDEN, "cont_hidden" }, + { LLM_TENSOR_CONT_OUTPUT, "cont_output" }, + { LLM_TENSOR_HC_ATTN_MIX, "blk.%d.hc_attn_mix" }, + { LLM_TENSOR_HC_ATTN_READ, "blk.%d.hc_attn_read" }, + { LLM_TENSOR_HC_ATTN_WRITE, "blk.%d.hc_attn_write" }, + { LLM_TENSOR_HC_FFN_MIX, "blk.%d.hc_ffn_mix" }, + { LLM_TENSOR_HC_FFN_READ, "blk.%d.hc_ffn_read" }, + { LLM_TENSOR_HC_FFN_WRITE, "blk.%d.hc_ffn_write" }, + { LLM_TENSOR_ATTN_PASS_LORA_DOWN, "blk.%d.attn_pass_lora_down" }, + { LLM_TENSOR_ATTN_PASS_LORA_UP, "blk.%d.attn_pass_lora_up" }, + { LLM_TENSOR_ATTN_PASS_GATE, "blk.%d.attn_pass_gate" }, + { LLM_TENSOR_FFN_EXP_EMBD, "blk.%d.ffn_exp_embd" }, { LLM_TENSOR_FFN_PRE_EXP_NORM, "blk.%d.ffn_pre_exp_norm" }, { LLM_TENSOR_FFN_POST_EXP_NORM, "blk.%d.ffn_post_exp_norm" }, { LLM_TENSOR_FFN_RESID_ALPHA, "blk.%d.ffn_resid_alpha" }, @@ -1158,6 +1195,36 @@ static const std::map LLM_TENSOR_INFOS = { // latent projections feed ggml_mul_mat, the buft probe must use MUL_MAT to keep them on GPU {LLM_TENSOR_FFN_LATENT_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, {LLM_TENSOR_FFN_LATENT_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_STREAM_EMBD, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_ADD}}, + {LLM_TENSOR_DEPTH_MEM_STATE_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_META_WRITE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_QUERY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_KEY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_VALUE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DEPTH_MEM_STREAM_GATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_STATE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_RETR, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_DIFF, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN_ROUTE, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_HIDDEN, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_CONT_OUTPUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_ATTN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_MIX, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_READ, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_HC_FFN_WRITE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_DOWN, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_LORA_UP, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_ATTN_PASS_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_FFN_EXP_EMBD, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_GET_ROWS}}, {LLM_TENSOR_FFN_PRE_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_POST_EXP_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_FFN_RESID_ALPHA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, @@ -1286,6 +1353,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) { case LLM_ARCH_LFM3: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_QWEN3NEXT: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_BAILINGMOE3: diff --git a/src/llama-arch.h b/src/llama-arch.h index b98be40cf29b..f811607b63de 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -106,6 +106,7 @@ enum llm_arch { LLM_ARCH_NEMOTRON, LLM_ARCH_NEMOTRON_H, LLM_ARCH_NEMOTRON_H_MOE, + LLM_ARCH_METIS16, LLM_ARCH_EXAONE, LLM_ARCH_EXAONE4, LLM_ARCH_EXAONE_MOE, @@ -276,6 +277,10 @@ enum llm_kv { LLM_KV_HRM_H_CYCLES, LLM_KV_HRM_L_CYCLES, LLM_KV_HRM_PREFIX_LM, + LLM_KV_METIS_PASS_COUNT, + LLM_KV_METIS_MEMORY_LENGTH, + LLM_KV_METIS_ROUTE_LENGTH, + LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, LLM_KV_REALTIME_DELAY_TOKENS, LLM_KV_MUDD_FEED_FORWARD_LENGTH, LLM_KV_MUDD_TAP_COUNT, @@ -333,6 +338,7 @@ enum llm_kv { LLM_KV_ATTENTION_KV_SOURCE_LAYERS, LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, LLM_KV_ATTENTION_OUTPUT_LORA_RANK, + LLM_KV_ATTENTION_PASS_LORA_RANK, LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, LLM_KV_ATTENTION_COMPRESS_RATIOS, LLM_KV_ATTENTION_SHARED_KV_LAYERS, @@ -391,6 +397,7 @@ enum llm_kv { LLM_KV_SSM_TIME_STEP_RANK, LLM_KV_SSM_GROUP_COUNT, LLM_KV_SSM_DT_B_C_RMS, + LLM_KV_SSM_NORM_EPS, LLM_KV_KDA_HEAD_DIM, LLM_KV_KDA_SAFE_GATE, @@ -566,6 +573,36 @@ enum llm_tensor { LLM_TENSOR_FFN_EXP_PROBS_B_VL, LLM_TENSOR_FFN_LATENT_DOWN, LLM_TENSOR_FFN_LATENT_UP, + LLM_TENSOR_STREAM_EMBD, + LLM_TENSOR_DEPTH_MEM_STATE_WRITE, + LLM_TENSOR_DEPTH_MEM_META_WRITE, + LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, + LLM_TENSOR_DEPTH_MEM_QUERY, + LLM_TENSOR_DEPTH_MEM_KEY, + LLM_TENSOR_DEPTH_MEM_VALUE, + LLM_TENSOR_DEPTH_MEM_OUT, + LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, + LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, + LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, + LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, + LLM_TENSOR_DEPTH_MEM_ROUTE, + LLM_TENSOR_DEPTH_MEM_STREAM_GATE, + LLM_TENSOR_CONT_HIDDEN_STATE, + LLM_TENSOR_CONT_HIDDEN_RETR, + LLM_TENSOR_CONT_HIDDEN_DIFF, + LLM_TENSOR_CONT_HIDDEN_ROUTE, + LLM_TENSOR_CONT_HIDDEN, + LLM_TENSOR_CONT_OUTPUT, + LLM_TENSOR_HC_ATTN_MIX, + LLM_TENSOR_HC_ATTN_READ, + LLM_TENSOR_HC_ATTN_WRITE, + LLM_TENSOR_HC_FFN_MIX, + LLM_TENSOR_HC_FFN_READ, + LLM_TENSOR_HC_FFN_WRITE, + LLM_TENSOR_ATTN_PASS_LORA_DOWN, + LLM_TENSOR_ATTN_PASS_LORA_UP, + LLM_TENSOR_ATTN_PASS_GATE, + LLM_TENSOR_FFN_EXP_EMBD, LLM_TENSOR_FFN_PRE_EXP_NORM, LLM_TENSOR_FFN_POST_EXP_NORM, LLM_TENSOR_FFN_RESID_ALPHA, diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0cd795a5690b..eb97569962a0 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2703,6 +2703,7 @@ uint32_t llama_context::graph_max_nodes(uint32_t n_tokens) const { res = std::max(n_tokens * 80, 64u * model.n_tensors()); } else if (model.arch == LLM_ARCH_QWEN3NEXT || model.arch == LLM_ARCH_KIMI_LINEAR || + model.arch == LLM_ARCH_METIS16 || model.arch == LLM_ARCH_BAILINGMOE3 || model.arch == LLM_ARCH_QWEN35 || model.arch == LLM_ARCH_QWEN35MOE || diff --git a/src/llama-hparams.h b/src/llama-hparams.h index 4d2cb89314d2..95daf482d287 100644 --- a/src/llama-hparams.h +++ b/src/llama-hparams.h @@ -209,6 +209,7 @@ struct llama_hparams { uint32_t ssm_d_state = 0; uint32_t ssm_dt_rank = 0; uint32_t ssm_n_group = 0; + float ssm_norm_eps = 0.0f; // 0 = use f_norm_rms_eps // for MiniMax-Text-01 linear attention uint32_t n_embd_head_la = 0; @@ -230,6 +231,13 @@ struct llama_hparams { uint32_t n_hrm_l_cycles = 0; bool hrm_prefix_lm = false; + // metis16 (passes over shared layers, depth memory) + uint32_t metis_n_pass = 0; + uint32_t metis_n_mem = 0; + uint32_t metis_n_route = 0; + uint32_t metis_pass_lora_rank = 0; + float metis_cont_limit = 0.0f; + // limite float f_xsa_eps = 0.0f; std::array final_logit_sigmoid_capping = {}; // scale, shift, temperature diff --git a/src/llama-model-saver.cpp b/src/llama-model-saver.cpp index 36f3850d5180..bfa1b547cbb5 100644 --- a/src/llama-model-saver.cpp +++ b/src/llama-model-saver.cpp @@ -247,6 +247,10 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_HRM_H_CYCLES, hparams.n_hrm_h_cycles); add_kv(LLM_KV_HRM_L_CYCLES, hparams.n_hrm_l_cycles); add_kv(LLM_KV_HRM_PREFIX_LM, hparams.hrm_prefix_lm); + add_kv(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + add_kv(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + add_kv(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); if (hparams.final_logit_sigmoid_capping[0] != 0.0f) { add_kv(LLM_KV_FINAL_LOGIT_SIGMOID_CAPPING, std::vector( hparams.final_logit_sigmoid_capping.begin(), hparams.final_logit_sigmoid_capping.end())); @@ -323,6 +327,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_ATTENTION_RECURRENT_LAYERS, hparams.is_recr_impl, true); add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, hparams.dsv4_o_group_count); add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, hparams.dsv4_o_lora_rank); + add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, hparams.dsv4_compress_rope_base); if (model->arch == LLM_ARCH_DEEPSEEK4 || hparams.dsv4_hc_mult > 0) { // the loader requires one compress ratio per layer, including nextn layers @@ -419,6 +424,7 @@ void llama_model_saver::add_kv_from_model() { add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms); + add_kv(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda); add_kv(LLM_KV_KDA_SAFE_GATE, hparams.kda_safe_gate); @@ -521,6 +527,32 @@ void llama_model_saver::add_tensors_from_model() { add_tensor(model->cls_out_b); add_tensor(model->cls_norm); add_tensor(model->hrm_z_l_init); + add_tensor(model->stream_embd); + add_tensor(model->depth_mem_state_write); + add_tensor(model->depth_mem_meta_write); + add_tensor(model->depth_mem_entry_bias); + add_tensor(model->depth_mem_query); + add_tensor(model->depth_mem_query_b); + add_tensor(model->depth_mem_key); + add_tensor(model->depth_mem_key_b); + add_tensor(model->depth_mem_value); + add_tensor(model->depth_mem_value_b); + add_tensor(model->depth_mem_out); + add_tensor(model->depth_mem_out_b); + add_tensor(model->depth_mem_route_state); + add_tensor(model->depth_mem_route_retr); + add_tensor(model->depth_mem_route_diff); + add_tensor(model->depth_mem_route_hist); + add_tensor(model->depth_mem_route_b); + add_tensor(model->depth_mem_stream_gate); + add_tensor(model->depth_mem_stream_gate_b); + add_tensor(model->cont_hidden_state); + add_tensor(model->cont_hidden_retr); + add_tensor(model->cont_hidden_diff); + add_tensor(model->cont_hidden_route); + add_tensor(model->cont_hidden_b); + add_tensor(model->cont_output); + add_tensor(model->cont_output_b); add_tensor(model->hc_head_fn); add_tensor(model->hc_head_base); add_tensor(model->hc_head_scale); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 87893bf671b6..3c4e598f1496 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -242,6 +242,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params return new llama_model_nemotron_h(params); case LLM_ARCH_NEMOTRON_H_MOE: return new llama_model_nemotron_h_moe(params); + case LLM_ARCH_METIS16: + return new llama_model_metis16(params); case LLM_ARCH_EXAONE: return new llama_model_exaone(params); case LLM_ARCH_EXAONE4: @@ -515,7 +517,7 @@ struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const str }; auto get_tensor_config = [&]() -> tensor_config { - if (ud->model->arch == LLM_ARCH_HRM_TEXT) { + if (ud->model->arch == LLM_ARCH_HRM_TEXT || ud->model->arch == LLM_ARCH_METIS16) { // aliased cache slots cannot satisfy the meta-split invariants, so replicate all tensors return {GGML_BACKEND_SPLIT_AXIS_MIRRORED, tensor, 0, 0}; } @@ -1049,6 +1051,7 @@ const char * llm_type_name(llm_type type) { case LLM_TYPE_7B: return "7B"; case LLM_TYPE_8B: return "8B"; case LLM_TYPE_9B: return "9B"; + case LLM_TYPE_10B: return "10B"; case LLM_TYPE_11B: return "11B"; case LLM_TYPE_12B: return "12B"; case LLM_TYPE_13B: return "13B"; @@ -3618,6 +3621,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { case LLM_ARCH_WAVTOKENIZER_DEC: case LLM_ARCH_NEMOTRON_H: case LLM_ARCH_NEMOTRON_H_MOE: + case LLM_ARCH_METIS16: case LLM_ARCH_KIMI_LINEAR: case LLM_ARCH_KIMI_K3: case LLM_ARCH_GLM5_NEXT: diff --git a/src/llama-model.h b/src/llama-model.h index da4964c98f7a..45d2b866624e 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -76,6 +76,7 @@ enum llm_type { LLM_TYPE_7B, LLM_TYPE_8B, LLM_TYPE_9B, + LLM_TYPE_10B, LLM_TYPE_11B, LLM_TYPE_12B, LLM_TYPE_13B, @@ -646,6 +647,18 @@ struct llama_layer { // gemma4 layer output scale, reused for talkie embedding skip scale struct ggml_tensor * out_scale = nullptr; + // metis16 per-pass hyper-connection mixes, attention pass LoRA, expert route embeddings + struct ggml_tensor * hc_attn_mix = nullptr; + struct ggml_tensor * hc_attn_read = nullptr; + struct ggml_tensor * hc_attn_write = nullptr; + struct ggml_tensor * hc_ffn_mix = nullptr; + struct ggml_tensor * hc_ffn_read = nullptr; + struct ggml_tensor * hc_ffn_write = nullptr; + struct ggml_tensor * attn_pass_lora_down = nullptr; + struct ggml_tensor * attn_pass_lora_up = nullptr; + struct ggml_tensor * attn_pass_gate = nullptr; + struct ggml_tensor * ffn_exp_embd = nullptr; + // limite struct ggml_tensor * attn_ve_gate = nullptr; struct ggml_tensor * attn_xsa_alpha = nullptr; @@ -779,6 +792,34 @@ struct llama_model { // hrm-text initial low-cycle state struct ggml_tensor * hrm_z_l_init = nullptr; + // metis16 hyper-connection stream embeddings, depth memory and continuation head + struct ggml_tensor * stream_embd = nullptr; + struct ggml_tensor * depth_mem_state_write = nullptr; + struct ggml_tensor * depth_mem_meta_write = nullptr; + struct ggml_tensor * depth_mem_entry_bias = nullptr; + struct ggml_tensor * depth_mem_query = nullptr; + struct ggml_tensor * depth_mem_query_b = nullptr; + struct ggml_tensor * depth_mem_key = nullptr; + struct ggml_tensor * depth_mem_key_b = nullptr; + struct ggml_tensor * depth_mem_value = nullptr; + struct ggml_tensor * depth_mem_value_b = nullptr; + struct ggml_tensor * depth_mem_out = nullptr; + struct ggml_tensor * depth_mem_out_b = nullptr; + struct ggml_tensor * depth_mem_route_state = nullptr; + struct ggml_tensor * depth_mem_route_retr = nullptr; + struct ggml_tensor * depth_mem_route_diff = nullptr; + struct ggml_tensor * depth_mem_route_hist = nullptr; + struct ggml_tensor * depth_mem_route_b = nullptr; + struct ggml_tensor * depth_mem_stream_gate = nullptr; + struct ggml_tensor * depth_mem_stream_gate_b = nullptr; + struct ggml_tensor * cont_hidden_state = nullptr; + struct ggml_tensor * cont_hidden_retr = nullptr; + struct ggml_tensor * cont_hidden_diff = nullptr; + struct ggml_tensor * cont_hidden_route = nullptr; + struct ggml_tensor * cont_hidden_b = nullptr; + struct ggml_tensor * cont_output = nullptr; + struct ggml_tensor * cont_output_b = nullptr; + // DeepSeek-V4 struct ggml_tensor * hc_head_fn = nullptr; struct ggml_tensor * hc_head_base = nullptr; diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c247858188..d62f7d5e998b 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -361,6 +361,16 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param quantize &= name.find("indexer.k_proj.weight") == std::string::npos; quantize &= name.find("indexer.q_proj.weight") == std::string::npos; + // metis16: small control tensors that steer every layer; the stream and hc_ tables (ne[0] = 4) would fall back to F16 + if (arch == LLM_ARCH_METIS16) { + quantize &= name.rfind("depth_mem_", 0) == std::string::npos; + quantize &= name.rfind("cont_", 0) == std::string::npos; + quantize &= name.find("stream_embd") == std::string::npos; + quantize &= name.find("attn_pass_lora") == std::string::npos; + quantize &= name.find("ffn_exp_embd") == std::string::npos; + quantize &= name.find("hc_") == std::string::npos; + } + // glm5-next if (arch == LLM_ARCH_GLM5_NEXT) { quantize &= name.find("hc_") == std::string::npos; diff --git a/src/models/mamba-base.cpp b/src/models/mamba-base.cpp index 370de0f0141d..63af3113dce0 100644 --- a/src/models/mamba-base.cpp +++ b/src/models/mamba-base.cpp @@ -290,7 +290,11 @@ ggml_tensor * llm_build_mamba_base::build_mamba2_layer(llm_graph_input_rs * inp, // grouped RMS norm if (model.layers[il].ssm_norm) { y = ggml_reshape_4d(ctx0, y, d_inner / n_group, n_group, n_seq_tokens, n_seqs); - y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + if (hparams.ssm_norm_eps > 0.0f) { + y = ggml_mul(ctx0, ggml_rms_norm(ctx0, y, hparams.ssm_norm_eps), model.layers[il].ssm_norm); + } else { + y = build_norm(y, model.layers[il].ssm_norm, NULL, LLM_NORM_RMS, il); + } } y = ggml_reshape_2d(ctx0, y, d_inner, n_seq_tokens * n_seqs); diff --git a/src/models/metis16.cpp b/src/models/metis16.cpp new file mode 100644 index 000000000000..4e4b032b9add --- /dev/null +++ b/src/models/metis16.cpp @@ -0,0 +1,401 @@ +#include "models.h" + +#include + +void llama_model_metis16::load_arch_hparams(llama_model_loader & ml) { + ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv); + ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner); + ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state); + ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank); + ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group); + ml.get_key(LLM_KV_SSM_NORM_EPS, hparams.ssm_norm_eps); + + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); + + ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all); + ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp); + ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size); + + ml.get_key(LLM_KV_HYPER_CONNECTION_COUNT, hparams.dsv4_hc_mult); + ml.get_key(LLM_KV_METIS_PASS_COUNT, hparams.metis_n_pass); + ml.get_key(LLM_KV_METIS_MEMORY_LENGTH, hparams.metis_n_mem); + ml.get_key(LLM_KV_METIS_ROUTE_LENGTH, hparams.metis_n_route); + ml.get_key(LLM_KV_ATTENTION_PASS_LORA_RANK, hparams.metis_pass_lora_rank); + ml.get_key(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, hparams.metis_cont_limit); + + n_pass = hparams.metis_n_pass; + n_stream = hparams.dsv4_hc_mult; + n_mem = hparams.metis_n_mem; + n_route = hparams.metis_n_route; + cont_limit = hparams.metis_cont_limit; + + if (n_pass < 1 || hparams.n_layer() % n_pass != 0) { + throw std::runtime_error(format("metis16: %u layers do not split into %u passes", hparams.n_layer(), n_pass)); + } + n_layer_phys = hparams.n_layer() / n_pass; + + bool has_attn = false; + for (uint32_t il = 0; il < hparams.n_layer(); ++il) { + if (hparams.n_head_kv(il) != hparams.n_head_kv(il % n_layer_phys)) { + throw std::runtime_error(format("metis16: layer %u does not repeat the layout of layer %u", il, il % n_layer_phys)); + } + hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0; + has_attn = has_attn || !hparams.is_recr_impl[il]; + } + if (!has_attn) { + throw std::runtime_error("metis16: at least one attention layer is required"); + } + + switch (n_layer_phys) { + case 12: type = LLM_TYPE_3B; break; // Praxis + case 20: type = LLM_TYPE_10B; break; // Logos + default: type = LLM_TYPE_UNKNOWN; + } +} + +void llama_model_metis16::load_arch_tensors(llama_model_loader &) { + LLAMA_LOAD_LOCALS; + + const int64_t d_conv = hparams.ssm_d_conv; + const int64_t d_inner = hparams.ssm_d_inner; + const int64_t d_state = hparams.ssm_d_state; + const int64_t n_ssm_head = hparams.ssm_dt_rank; + const int64_t n_group = hparams.ssm_n_group; + const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head; + const int64_t n_latent = hparams.moe_latent_size; + const int64_t n_ff_shexp = hparams.n_ff_shexp; + const int64_t n_s = n_stream; + const int64_t n_p = n_pass; + const int64_t n_m = n_mem; + const int64_t n_r = n_route; + + int64_t n_attn = 0; + for (uint32_t il = 0; il < n_layer_phys; ++il) { + n_attn += !hparams.is_recr(il); + } + + tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); + output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); + output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + if (output == NULL) { + output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); + } + + stream_embd = create_tensor(tn(LLM_TENSOR_STREAM_EMBD, "weight"), {n_s, n_embd}, 0); + + depth_mem_state_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STATE_WRITE, "weight"), {n_embd, n_m}, 0); + depth_mem_meta_write = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_META_WRITE, "weight"), {n_r + 3, n_m}, 0); + depth_mem_entry_bias = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ENTRY_BIAS, "weight"), {n_m, n_p*(n_attn + 1)}, 0); + depth_mem_query = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "weight"), {n_embd, n_m}, 0); + depth_mem_query_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_QUERY, "bias"), {n_m}, 0); + depth_mem_key = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "weight"), {n_m, n_m}, 0); + depth_mem_key_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_KEY, "bias"), {n_m}, 0); + depth_mem_value = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "weight"), {n_m, n_m}, 0); + depth_mem_value_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_VALUE, "bias"), {n_m}, 0); + depth_mem_out = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "weight"), {n_m, n_embd}, 0); + depth_mem_out_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_OUT, "bias"), {n_embd}, 0); + depth_mem_route_state = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_STATE, "weight"), {n_embd, n_r}, 0); + depth_mem_route_retr = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_RETR, "weight"), {n_embd, n_r}, 0); + depth_mem_route_diff = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_DIFF, "weight"), {n_embd, n_r}, 0); + depth_mem_route_hist = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE_HIST, "weight"), {n_r, n_r}, 0); + depth_mem_route_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_ROUTE, "bias"), {n_r}, 0); + depth_mem_stream_gate = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "weight"), {n_embd, n_s}, 0); + depth_mem_stream_gate_b = create_tensor(tn(LLM_TENSOR_DEPTH_MEM_STREAM_GATE, "bias"), {n_s}, 0); + + cont_hidden_state = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_STATE, "weight"), {n_embd, n_r}, 0); + cont_hidden_retr = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_RETR, "weight"), {n_embd, n_r}, 0); + cont_hidden_diff = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_DIFF, "weight"), {n_embd, n_r}, 0); + cont_hidden_route = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN_ROUTE, "weight"), {n_r, n_r}, 0); + cont_hidden_b = create_tensor(tn(LLM_TENSOR_CONT_HIDDEN, "bias"), {n_r}, 0); + cont_output = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "weight"), {n_r, 1}, 0); + cont_output_b = create_tensor(tn(LLM_TENSOR_CONT_OUTPUT, "bias"), {1}, 0); + + for (int il = 0; il < (int) n_layer_phys; ++il) { + auto & layer = layers[il]; + + layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", il), {n_embd}, 0); + layer.hc_attn_mix = create_tensor(tn(LLM_TENSOR_HC_ATTN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_attn_read = create_tensor(tn(LLM_TENSOR_HC_ATTN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_attn_write = create_tensor(tn(LLM_TENSOR_HC_ATTN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_mix = create_tensor(tn(LLM_TENSOR_HC_FFN_MIX, "weight", il), {n_s, n_s, n_p}, 0); + layer.hc_ffn_read = create_tensor(tn(LLM_TENSOR_HC_FFN_READ, "weight", il), {n_s, n_p}, 0); + layer.hc_ffn_write = create_tensor(tn(LLM_TENSOR_HC_FFN_WRITE, "weight", il), {n_s, n_p}, 0); + layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", il), {n_embd}, 0); + + if (hparams.is_recr(il)) { + layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", il), {n_embd, d_in_proj}, 0); + layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", il), {d_conv, d_inner + 2*n_group*d_state}, 0); + layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", il), {d_inner + 2*n_group*d_state}, 0); + layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", il), {n_ssm_head}, 0); + layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", il), {d_inner / n_group, n_group}, 0); + layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", il), {d_inner, n_embd}, 0); + + // no "weight" suffix for these + layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, il), {1, n_ssm_head}, 0); + layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, il), {1, n_ssm_head}, 0); + } else { + const int64_t n_qkv = n_embd_head_k * (hparams.n_head(il) + 2*hparams.n_head_kv(il)); + const int64_t rank = hparams.metis_pass_lora_rank; + + layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", il), {n_embd, n_qkv}, 0); + layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", il), {n_embd_head_k * hparams.n_head(il), n_embd}, 0); + layer.attn_pass_lora_down = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_DOWN, "weight", il), {n_embd, rank}, 0); + layer.attn_pass_lora_up = create_tensor(tn(LLM_TENSOR_ATTN_PASS_LORA_UP, "weight", il), {rank, n_qkv}, 0); + layer.attn_pass_gate = create_tensor(tn(LLM_TENSOR_ATTN_PASS_GATE, "weight", il), {n_p}, 0); + } + + const int64_t n_ff_exp = hparams.n_ff_exp(il); + + layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", il), {n_embd, n_latent}, 0); + layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", il), {n_latent, n_embd}, 0); + layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", il), {n_latent + n_r, n_expert}, 0); + layer.ffn_gate_inp_b = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "bias", il), {n_expert}, 0); + layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", il), {n_expert}, 0); + layer.ffn_exp_embd = create_tensor(tn(LLM_TENSOR_FFN_EXP_EMBD, "weight", il), {n_r, n_expert}, 0); + layer.ffn_gate_up_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_UP_EXPS, "weight", il), {n_latent, 2*n_ff_exp, n_expert}, 0); + layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", il), {n_ff_exp, n_latent, n_expert}, 0); + layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", il), {n_latent, n_ff_shexp}, 0); + layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", il), {n_ff_shexp, n_latent}, 0); + } + + // the virtual layers of later passes share the physical layer's weights + for (int il = n_layer_phys; il < n_layer; ++il) { + layers[il] = layers[il % n_layer_phys]; + } +} + +std::unique_ptr llama_model_metis16::build_arch_graph(const llm_graph_params & params) const { + return std::make_unique(*this, params); +} + +// streams are kept stream-innermost: [n_stream, n_embd, n_tokens] +ggml_tensor * llama_model_metis16::graph::stream_mean(ggml_tensor * x) const { + ggml_tensor * mean = ggml_mean(ctx0, ggml_reshape_2d(ctx0, x, x->ne[0], x->ne[1]*x->ne[2])); + return ggml_reshape_2d(ctx0, mean, x->ne[1], x->ne[2]); +} + +// returns the normalized sublayer input +ggml_tensor * llama_model_metis16::graph::hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, + int p, ggml_tensor ** mixed, int il) const { + const int64_t n_s = x->ne[0]; + ggml_tensor * x2 = ggml_reshape_2d(ctx0, x, n_s, x->ne[1]*x->ne[2]); + + ggml_tensor * m = ggml_view_2d(ctx0, mix, n_s, n_s, mix->nb[1], p*mix->nb[2]); + *mixed = ggml_mul_mat(ctx0, m, x2); + + ggml_tensor * r = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, read, n_s, p*read->nb[1]), n_s, 1); + ggml_tensor * src = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, r, x2), x->ne[1], x->ne[2]); + return build_norm(src, norm, NULL, LLM_NORM_RMS, il); +} + +ggml_tensor * llama_model_metis16::graph::hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const { + const int64_t n_s = mixed->ne[0]; + ggml_tensor * w = ggml_reshape_2d(ctx0, ggml_view_1d(ctx0, write, n_s, p*write->nb[1]), 1, n_s); + ggml_tensor * u = ggml_reshape_2d(ctx0, update, 1, update->ne[0]*update->ne[1]); + ggml_tensor * out = ggml_add(ctx0, mixed, ggml_mul_mat(ctx0, w, u)); + return ggml_reshape_3d(ctx0, out, n_s, update->ne[0], update->ne[1]); +} + +// blends the retrieved entries into the streams; returns their summary +ggml_tensor * llama_model_metis16::graph::mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const { + const int64_t n_s = (*x)->ne[0]; + const int64_t n_m = model.n_mem; + + ggml_tensor * x_e = ggml_cont(ctx0, ggml_permute(ctx0, *x, 1, 0, 2, 3)); + + ggml_tensor * q = ggml_mul_mat(ctx0, model.depth_mem_query, ggml_reshape_2d(ctx0, x_e, n_embd, n_s*n_tokens)); + q = ggml_reshape_3d(ctx0, ggml_add(ctx0, q, model.depth_mem_query_b), n_m, n_s, n_tokens); + + ggml_tensor * kq = ggml_mul_mat(ctx0, bank_k, q); + kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(n_m)), 0.0f); + + ggml_tensor * ret = ggml_mul_mat(ctx0, bank_vt, kq); + ggml_tensor * proj = ggml_mul_mat(ctx0, model.depth_mem_out, ggml_reshape_2d(ctx0, ret, n_m, n_s*n_tokens)); + proj = ggml_reshape_3d(ctx0, ggml_add(ctx0, proj, model.depth_mem_out_b), n_embd, n_s, n_tokens); + + ggml_tensor * gate = ggml_sum_rows(ctx0, ggml_mul(ctx0, x_e, model.depth_mem_stream_gate)); + gate = ggml_sigmoid(ctx0, ggml_add(ctx0, gate, ggml_reshape_2d(ctx0, model.depth_mem_stream_gate_b, 1, n_s))); + + ggml_tensor * proj_s = ggml_cont(ctx0, ggml_permute(ctx0, proj, 1, 0, 2, 3)); + *x = ggml_add(ctx0, *x, ggml_mul(ctx0, proj_s, ggml_reshape_3d(ctx0, gate, n_s, 1, n_tokens))); + + return stream_mean(proj_s); +} + +void llama_model_metis16::graph::mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry) { + const int64_t n_m = model.n_mem; + + if (cont == nullptr) { + cont = ggml_scale(ctx0, confidence, 0.0f); + } + ggml_tensor * meta = ggml_concat(ctx0, ggml_concat(ctx0, summary, entropy, 0), ggml_concat(ctx0, confidence, cont, 0), 0); + + ggml_tensor * e = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_state_write, stream_mean(x)), ggml_mul_mat(ctx0, model.depth_mem_meta_write, meta)); + e = ggml_add(ctx0, e, ggml_view_1d(ctx0, model.depth_mem_entry_bias, n_m, entry*model.depth_mem_entry_bias->nb[1])); + + ggml_tensor * k = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_key, e), model.depth_mem_key_b); + ggml_tensor * v = ggml_add(ctx0, ggml_mul_mat(ctx0, model.depth_mem_value, e), model.depth_mem_value_b); + k = ggml_reshape_3d(ctx0, k, n_m, 1, n_tokens); + v = ggml_reshape_3d(ctx0, v, 1, n_m, n_tokens); + bank_k = bank_k ? ggml_concat(ctx0, bank_k, k, 1) : k; + bank_vt = bank_vt ? ggml_concat(ctx0, bank_vt, v, 0) : v; +} + +ggml_tensor * llama_model_metis16::graph::route_features(const llama_model_metis16 & model, ggml_tensor * state, + ggml_tensor * summary, ggml_tensor * diff, ggml_tensor * history) const { + ggml_tensor * f = ggml_mul_mat(ctx0, model.depth_mem_route_state, state); + if (summary) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_retr, summary)); + } + if (diff) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_diff, diff)); + } + if (history) { + f = ggml_add(ctx0, f, ggml_mul_mat(ctx0, model.depth_mem_route_hist, history)); + } + return ggml_tanh(ctx0, ggml_add(ctx0, f, model.depth_mem_route_b)); +} + +llama_model_metis16::graph::graph(const llama_model_metis16 & model, const llm_graph_params & params) : + llm_build_mamba_base(params) { + const int64_t n_s = model.n_stream; + const int n_p = model.n_pass; + const int n_phys = model.n_layer_phys; + const int64_t n_r = model.n_route; + const int64_t n_hd = hparams.n_embd_head_k(); + const int64_t n_exp = hparams.n_expert; + const int64_t n_used = hparams.n_expert_used(); + + ggml_tensor * inpL = build_inp_embd(model.tok_embd); + auto * inp = build_inp_mem_hybrid(); + ggml_tensor * inp_out_ids = build_inp_out_ids(); + + ggml_tensor * x = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, inpL, 1, n_embd, n_tokens), n_s, n_embd, n_tokens, 1); + x = ggml_add(ctx0, x, model.stream_embd); + cb(x, "streams_in", -1); + + ggml_tensor * prev_state = stream_mean(x); + ggml_tensor * history = nullptr; // routing history, [n_route, n_tokens] + ggml_tensor * cont = nullptr; // depth head reading of the last pass, [1, n_tokens] + int n_entry = 0; + + for (int p = 0; p < n_p; ++p) { + ggml_tensor * summ = nullptr; + ggml_tensor * ent = nullptr; + ggml_tensor * conf = nullptr; + + for (int il_p = 0; il_p < n_phys; ++il_p) { + const int il = p*n_phys + il_p; + const auto & layer = model.layers[il]; + const bool is_attn = !hparams.is_recr(il); + const bool writes = is_attn || il_p == n_phys - 1; + + ggml_tensor * retrieved = bank_k ? mem_read(model, &x) : nullptr; + ggml_tensor * feats = route_features(model, stream_mean(x), retrieved, nullptr, history); + + ggml_tensor * mixed = nullptr; + ggml_tensor * cur = hc_read(x, layer.hc_attn_mix, layer.hc_attn_read, layer.attn_norm, p, &mixed, il); + if (is_attn) { + ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur); + ggml_tensor * lora = ggml_mul_mat(ctx0, layer.attn_pass_lora_up, ggml_mul_mat(ctx0, layer.attn_pass_lora_down, cur)); + qkv = ggml_add(ctx0, qkv, ggml_mul(ctx0, lora, ggml_view_1d(ctx0, layer.attn_pass_gate, 1, p*layer.attn_pass_gate->nb[0]))); + + const int64_t n_head = hparams.n_head(il); + const int64_t n_head_kv = hparams.n_head_kv(il); + const size_t es = ggml_element_size(qkv); + ggml_tensor * Q = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head, n_tokens, n_hd*es, qkv->nb[1], 0)); + ggml_tensor * K = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*n_head*es)); + ggml_tensor * V = ggml_cont(ctx0, ggml_view_3d(ctx0, qkv, n_hd, n_head_kv, n_tokens, n_hd*es, qkv->nb[1], n_hd*(n_head + n_head_kv)*es)); + + cur = build_attn(inp->get_attn(), layer.wo, nullptr, nullptr, Q, K, V, nullptr, nullptr, nullptr, 1.0f/sqrtf(float(n_hd)), il); + } else { + cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il); + } + cb(cur, "mixer_out", il); + x = hc_write(mixed, layer.hc_attn_write, cur, p); + + cur = hc_read(x, layer.hc_ffn_mix, layer.hc_ffn_read, layer.ffn_norm, p, &mixed, il); + { + ggml_tensor * latent = ggml_mul_mat(ctx0, layer.ffn_latent_down, cur); + ggml_tensor * logits = ggml_mul_mat(ctx0, layer.ffn_gate_inp, ggml_concat(ctx0, latent, feats, 0)); + logits = ggml_add(ctx0, logits, layer.ffn_gate_inp_b); + + // the balance bias picks the experts; the weights come from the unbiased logits + ggml_tensor * selected = ggml_argsort_top_k(ctx0, ggml_add(ctx0, logits, layer.ffn_exp_probs_b), n_used); + + ggml_tensor * moe_out = build_moe_ffn(latent, nullptr, nullptr, nullptr, layer.ffn_down_exps, nullptr, + n_exp, n_used, LLM_FFN_SILU, false, 0.0f, LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT, il_p, + logits, layer.ffn_gate_up_exps, nullptr, nullptr, nullptr, selected); + ggml_tensor * shexp = build_ffn(latent, + layer.ffn_up_shexp, NULL, NULL, + layer.ffn_gate_shexp, NULL, NULL, + layer.ffn_down_shexp, NULL, NULL, + NULL, LLM_FFN_SILU, LLM_FFN_PAR, il_p); + cur = ggml_mul_mat(ctx0, layer.ffn_latent_up, ggml_add(ctx0, moe_out, shexp)); + cb(cur, "ffn_out", il); + + // routing summary: the chosen experts' identity vectors blended by their weights + ggml_tensor * logits3 = ggml_reshape_3d(ctx0, logits, 1, n_exp, n_tokens); + ggml_tensor * weights = ggml_soft_max(ctx0, ggml_reshape_2d(ctx0, ggml_get_rows(ctx0, logits3, selected), n_used, n_tokens)); + ggml_tensor * rows = ggml_get_rows(ctx0, layer.ffn_exp_embd, ggml_reshape_1d(ctx0, ggml_cont(ctx0, selected), n_used*n_tokens)); + rows = ggml_cont(ctx0, ggml_permute(ctx0, ggml_reshape_3d(ctx0, rows, n_r, n_used, n_tokens), 1, 0, 2, 3)); + summ = ggml_reshape_2d(ctx0, ggml_mul_mat(ctx0, rows, ggml_reshape_3d(ctx0, weights, n_used, 1, n_tokens)), n_r, n_tokens); + + if (writes) { + ggml_tensor * probs = ggml_soft_max(ctx0, logits); + ent = ggml_sum_rows(ctx0, ggml_mul(ctx0, probs, ggml_log(ctx0, ggml_clamp(ctx0, probs, 1.0e-9f, 1.0f)))); + ent = ggml_scale(ctx0, ent, -1.0f); + conf = ggml_get_rows(ctx0, ggml_reshape_3d(ctx0, probs, 1, n_exp, n_tokens), + ggml_reshape_2d(ctx0, ggml_argmax(ctx0, probs), 1, n_tokens)); + conf = ggml_reshape_2d(ctx0, conf, 1, n_tokens); + } + } + x = hc_write(mixed, layer.hc_ffn_write, cur, p); + cb(x, "streams", il); + + history = history ? ggml_add(ctx0, ggml_scale(ctx0, history, 0.5f), ggml_scale(ctx0, summ, 0.5f)) : ggml_scale(ctx0, summ, 0.5f); + + if (is_attn) { + mem_write(model, x, summ, ent, conf, cont, n_entry++); + } + } + + // end of the pass: one more memory read, the depth head, and an entry recording where the pass left the token + ggml_tensor * retrieved = mem_read(model, &x); + ggml_tensor * state = stream_mean(x); + ggml_tensor * diff = ggml_sub(ctx0, state, prev_state); + ggml_tensor * feats = route_features(model, state, retrieved, diff, history); + + ggml_tensor * h = ggml_mul_mat(ctx0, model.cont_hidden_state, state); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_retr, retrieved)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_diff, diff)); + h = ggml_add(ctx0, h, ggml_mul_mat(ctx0, model.cont_hidden_route, feats)); + h = ggml_silu(ctx0, ggml_add(ctx0, h, model.cont_hidden_b)); + + ggml_tensor * logit = ggml_add(ctx0, ggml_mul_mat(ctx0, model.cont_output, h), model.cont_output_b); + ggml_tensor * scale = ggml_clamp(ctx0, ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, state))), 1.0e-6f, INFINITY); + ggml_tensor * innov = ggml_sqrt(ctx0, ggml_mean(ctx0, ggml_sqr(ctx0, diff))); + logit = ggml_add(ctx0, logit, ggml_scale_bias(ctx0, ggml_tanh(ctx0, ggml_div(ctx0, innov, scale)), 1.0f, -0.5f)); + cont = ggml_sigmoid(ctx0, ggml_clamp(ctx0, logit, -model.cont_limit, model.cont_limit)); + cb(cont, "continuation", p); + + mem_write(model, x, summ, ent, conf, cont, n_entry++); + prev_state = state; + } + + ggml_tensor * cur = stream_mean(x); + if (inp_out_ids) { + cur = ggml_get_rows(ctx0, cur, inp_out_ids); + } + + cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); + cb(cur, "result_norm", -1); + res->t_embd = cur; + + cur = build_lora_mm(model.output, cur); + cb(cur, "result_output", -1); + res->t_logits = cur; + +} diff --git a/src/models/models.h b/src/models/models.h index 1fc7cae1dff0..dd3b7b1edfbb 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -1799,6 +1799,39 @@ struct llama_model_nemotron_h_moe : public llama_model_nemotron_h { }; +struct llama_model_metis16 : public llama_model_base { + llama_model_metis16(const struct llama_model_params & params) : llama_model_base(params) {} + void load_arch_hparams(llama_model_loader & ml) override; + void load_arch_tensors(llama_model_loader & ml) override; + + uint32_t n_pass = 1; + uint32_t n_stream = 1; + uint32_t n_mem = 0; + uint32_t n_route = 0; + uint32_t n_layer_phys = 0; + float cont_limit = 0.0f; + + struct graph : public llm_build_mamba_base { + graph(const llama_model_metis16 & model, const llm_graph_params & params); + + // memory bank of the current forward: keys [n_mem, n_entry, n_tokens], values [n_entry, n_mem, n_tokens] + ggml_tensor * bank_k = nullptr; + ggml_tensor * bank_vt = nullptr; + + ggml_tensor * stream_mean(ggml_tensor * x) const; + ggml_tensor * hc_read(ggml_tensor * x, ggml_tensor * mix, ggml_tensor * read, ggml_tensor * norm, int p, ggml_tensor ** mixed, int il) const; + ggml_tensor * hc_write(ggml_tensor * mixed, ggml_tensor * write, ggml_tensor * update, int p) const; + ggml_tensor * mem_read(const llama_model_metis16 & model, ggml_tensor ** x) const; + void mem_write(const llama_model_metis16 & model, ggml_tensor * x, ggml_tensor * summary, + ggml_tensor * entropy, ggml_tensor * confidence, ggml_tensor * cont, int entry); + ggml_tensor * route_features(const llama_model_metis16 & model, ggml_tensor * state, ggml_tensor * summary, + ggml_tensor * diff, ggml_tensor * history) const; + }; + + std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; +}; + + struct llama_model_exaone : public llama_model_base { llama_model_exaone(const struct llama_model_params & params) : llama_model_base(params) {} void load_arch_hparams(llama_model_loader & ml) override; diff --git a/tests/fusion/MTL.csv b/tests/fusion/MTL.csv index 254b8a03ae96..eda5119b5e83 100644 --- a/tests/fusion/MTL.csv +++ b/tests/fusion/MTL.csv @@ -188,6 +188,8 @@ maple ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, mellum ,0 ,any ,MUL+ADD , 4 mellum ,0 ,any ,RMS_NORM+MUL , 9 mellum ,0 ,any ,SOFT_MAX+ARGSORT+GET_ROWS+SUM_ROWS+CLAMP+DIV, 2 +metis16 ,0 ,any ,MUL+ADD , 8 +metis16 ,0 ,any ,RMS_NORM+MUL , 11 mimo2 ,0 ,any ,MUL+ADD , 4 mimo2 ,0 ,any ,RMS_NORM+MUL , 5 minicpm ,0 ,any ,RMS_NORM+MUL , 5 diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index 32569dc86c70..fe949f2c2afe 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -295,6 +295,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { n_ff = 192; } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { n_layer = 3; + } else if (arch == LLM_ARCH_METIS16) { + n_layer = 4; // 2 passes over 2 physical layers, one attention and one Mamba-2 } else if (arch == LLM_ARCH_CHAMELEON) { n_vocab = 10240; } else if (arch == LLM_ARCH_QWEN3TTS) { @@ -369,6 +371,15 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer); } ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer); + } else if (arch == LLM_ARCH_METIS16) { + // the layer types repeat in every pass + std::vector n_head_kv_per_layer; + n_head_kv_per_layer.reserve(n_layer); + for (uint32_t il = 0; il < n_layer; il++) { + n_head_kv_per_layer.push_back(il % 2 == 0 ? n_head_kv : 0); + } + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); + ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_kv_per_layer); } else { ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head); ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, (arch == LLM_ARCH_DEEPSEEK4 || arch == LLM_ARCH_DEEPSEEK41) ? uint32_t(1) : n_head_kv); @@ -616,6 +627,17 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f); } + if (arch == LLM_ARCH_METIS16) { + ms.add_kv(LLM_KV_METIS_PASS_COUNT, uint32_t(2)); + ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4)); + ms.add_kv(LLM_KV_METIS_MEMORY_LENGTH, uint32_t(64)); + ms.add_kv(LLM_KV_METIS_ROUTE_LENGTH, uint32_t(32)); + ms.add_kv(LLM_KV_ATTENTION_PASS_LORA_RANK, uint32_t(8)); + ms.add_kv(LLM_KV_METIS_CONTINUATION_LOGIT_LIMIT, 4.0f); + ms.add_kv(LLM_KV_SSM_NORM_EPS, 1e-5f); + ms.add_kv(LLM_KV_MOE_LATENT_SIZE, n_embd/2); + } + if (arch == LLM_ARCH_LIMITE) { ms.add_kv(LLM_KV_ATTENTION_SCALE, 0.1f); ms.add_kv(LLM_KV_ATTENTION_XSA_EPS, 1e-4f); @@ -972,6 +994,7 @@ static bool moe_mandatory(const llm_arch arch) { case LLM_ARCH_MAPLE: case LLM_ARCH_MUSE_SPARK: case LLM_ARCH_KOLIBRI1: + case LLM_ARCH_METIS16: return true; default: return false; From a464960ed34564913a94afab17bf79fde727023e Mon Sep 17 00:00:00 2001 From: Aaron Teo Date: Tue, 3 Nov 2026 23:51:07 +0800 Subject: [PATCH 028/286] ggml-zdnn: fix fattn test failure (#31583) --- ggml/src/ggml-zdnn/fattn.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-zdnn/fattn.cpp b/ggml/src/ggml-zdnn/fattn.cpp index 7c7801cbb1dc..e877b7a75a27 100644 --- a/ggml/src/ggml-zdnn/fattn.cpp +++ b/ggml/src/ggml-zdnn/fattn.cpp @@ -20,8 +20,8 @@ bool ggml_zdnn_supports_flash_attn_ext(const ggml_backend_zdnn_device_context * memcpy(&max_bias, (const float *) op->op_params + 1, sizeof(float)); memcpy(&logit_softcap, (const float *) op->op_params + 2, sizeof(float)); - // sinks, alibi, softcap and sparse k/v sets are not implemented - if (op->src[4] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) { + // sinks, alibi, softcap, sparse k/v sets and k/v row lists are not implemented + if (op->src[4] != nullptr || op->src[5] != nullptr || max_bias != 0.0f || logit_softcap != 0.0f || ggml_get_op_params_i32(op, 4) != 0) { return false; } From 0f35d6d7b0f548993d4323eb1df8e7ca4cf039c5 Mon Sep 17 00:00:00 2001 From: randomized <286843573@qq.com> Date: Tue, 3 Nov 2026 23:53:57 +0800 Subject: [PATCH 029/286] spec : save and restore MTP hidden state (#30141) * speculative : save and restore MTP hidden state Assisted-by: Codex * tests : remove MTP state test Assisted-by: Codex * speculative : simplify MTP checkpoint state Save only the boundary hidden row; verification rows are rebuilt on the supported restore paths. Assisted-by: GPT-6 --------- Co-authored-by: zouyongxing --- common/speculative.cpp | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index 14fee2b2c501..f82dc879f1b8 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1967,6 +1967,29 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const size_t row_bytes = (size_t) n_embd * sizeof(float); std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes); } + + bool get_state(llama_seq_id seq_id, std::vector & data) const override { + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return false; + } + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + data.resize(row_bytes); + std::memcpy(data.data(), pending_h[seq_id].data(), row_bytes); + return true; + } + + void set_state(llama_seq_id seq_id, const std::vector & data) override { + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return; + } + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + if (data.size() != row_bytes) { + return; + } + std::memcpy(pending_h[seq_id].data(), data.data(), row_bytes); + } }; // state of self-speculation (simple implementation, not ngram-map) From 20a51f24a8523b290dcc1fc4fe9ce8fbdef9de13 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Tue, 3 Nov 2026 08:26:18 -0800 Subject: [PATCH 030/286] hexagon: add DSV4_HC_PRE and DSV4_HC_POST ops (#30953) * hexagon: add support for DSV4_HC_PRE * hexagon: hsv4-hc-pre plain/gated split kernel and l2 prefetch optimization * hex-dsv4-hc-pre: double-buffered DMA for prefill * hex-dsv4-hc-pre: refactor DMA block setup * hex-dsv4-hc-pre: precompute kernel params * hex-dsv4-hc-pre: support for n_hc != 4 * hexagon: add support for DSV4_HC_POST * hex-dsv4-hc-pre: refactor gated PRE prefill DMA to dst writeback after each block * hex-dsv4-hc-pre: remove in-loop vector spills and redundant device-side checks * hex-run: add support for test-llama-archs tool * hex-dsv4: refactoring and optimizations Remove non-dma paths. Clean up DMA pipelines and dma_addr_t casts. Add dims collapsing support. Tested with test-llama-archs -a qwen4ext and test-backend-ops --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/htp-kparams.cpp | 434 +++++++++++++++ ggml/src/ggml-hexagon/htp-kparams.h | 1 + ggml/src/ggml-hexagon/htp-opnode.cpp | 11 + ggml/src/ggml-hexagon/htp/CMakeLists.txt | 1 + ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c | 675 +++++++++++++++++++++++ ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h | 77 +++ ggml/src/ggml-hexagon/htp/glu-ops.c | 47 -- ggml/src/ggml-hexagon/htp/htp-ctx.h | 27 +- ggml/src/ggml-hexagon/htp/htp-ops.h | 2 + ggml/src/ggml-hexagon/htp/hvx-sigmoid.h | 47 ++ ggml/src/ggml-hexagon/htp/main.c | 7 + scripts/snapdragon/run.py | 54 +- tests/test-backend-ops.cpp | 17 + 13 files changed, 1328 insertions(+), 72 deletions(-) create mode 100644 ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index e1d3385e2210..f5c4ad7cb116 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -53,6 +53,8 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_OP_FILL: return HTP_OP_FILL; case GGML_OP_DIAG: return HTP_OP_DIAG; case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; + case GGML_OP_DSV4_HC_PRE: return HTP_OP_DSV4_HC_PRE; + case GGML_OP_DSV4_HC_POST: return HTP_OP_DSV4_HC_POST; case GGML_OP_TRI: return HTP_OP_TRI; case GGML_OP_PAD: return HTP_OP_PAD; case GGML_OP_IM2COL: return HTP_OP_IM2COL; @@ -4141,6 +4143,382 @@ static bool htp_kparams_solve_conv2d_dw( return true; } +static bool htp_dsv4_hc_pre_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst +) { + if (!x || !weights || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (x->ne[1] != 4 || dst->ne[0] != x->ne[0]) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + const int64_t x_tokens = x->ne[2] * x->ne[3]; + const int64_t d_tokens = dst->ne[1] * dst->ne[2] * dst->ne[3]; + const int64_t w_tokens = gated ? (weights->ne[2] * weights->ne[3]) : (weights->ne[1] * weights->ne[2] * weights->ne[3]); + + if (x_tokens != d_tokens || x_tokens != w_tokens) { + return false; + } + + if (x->ne[3] == 1 && dst->ne[2] == 1 && dst->ne[3] == 1 && + (gated ? (weights->ne[3] == 1) : (weights->ne[2] == 1 && weights->ne[3] == 1))) { + return false; + } + + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (dst->ne[2] > 1 && dst->nb[2] != dst->ne[1] * dst->nb[1]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (gated) { + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } else { + if (weights->ne[2] > 1 && weights->nb[2] != weights->ne[1] * weights->nb[1]) return false; + if (weights->ne[3] > 1 && weights->nb[3] != weights->ne[2] * weights->nb[2]) return false; + } + + return true; +} + +static void htp_dsv4_hc_pre_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * w_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[2] * x->ne[3]; + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + + *x_c = *x; + x_c->ne[2] = n_tokens; + x_c->ne[3] = 1; + x_c->nb[3] = x_c->ne[2] * x_c->nb[2]; + + *d_c = *dst; + d_c->ne[1] = n_tokens; + d_c->ne[2] = 1; + d_c->ne[3] = 1; + d_c->nb[2] = d_c->ne[1] * d_c->nb[1]; + d_c->nb[3] = d_c->nb[2]; + + *w_c = *weights; + if (gated) { + w_c->ne[2] = n_tokens; + w_c->ne[3] = 1; + w_c->nb[3] = w_c->ne[2] * w_c->nb[2]; + } else { + w_c->ne[1] = n_tokens; + w_c->ne[2] = 1; + w_c->ne[3] = 1; + w_c->nb[2] = w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_pre_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * weights, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || weights->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[1] != 4 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != x->ne[2] || dst->ne[2] != 1 || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || weights->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(weights) || ggml_is_permuted(dst)) { + return false; + } + + const bool gated = ggml_get_op_params_i32(dst, 1) != 0; + if (gated) { + if (weights->ne[0] != x->ne[0] || weights->ne[1] != x->ne[1] || weights->ne[2] != x->ne[2] || weights->ne[3] != 1) { + return false; + } + } else { + if (weights->ne[0] != x->ne[1] || weights->ne[1] != x->ne[2] || weights->ne[2] != 1 || weights->ne[3] != 1) { + return false; + } + } + + const uint32_t n_tokens = (uint32_t) x->ne[2]; + kparams->kernel_type = gated ? HTP_DSV4_HC_PRE_KERNEL_GATED : HTP_DSV4_HC_PRE_KERNEL_PLAIN; + + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs); + kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + const uint32_t n_rows = gated ? HTP_DSV4_HC_PRE_GATED_ROWS : HTP_DSV4_HC_PRE_PLAIN_ROWS; + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = n_rows * stride; + if (2 * buf_sz * kparams->n_threads <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = n_rows * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_pre( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_pre_kernel_params * kparams +) { + if (htp_dsv4_hc_pre_can_collapse(op->src[0], op->src[1], op)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(op->src[0], op->src[1], op, &x_c, &w_c, &d_c); + return htp_kparams_solve_dsv4_hc_pre_impl(sess, &x_c, &w_c, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_pre_impl(sess, op->src[0], op->src[1], op, kparams); +} + +static bool htp_dsv4_hc_post_can_collapse( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst +) { + if (!x || !residual || !post || !dst) { + return false; + } + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || post->ne[0] != 4) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4)) { + return false; + } + + const int64_t x_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + const int64_t r_tokens = residual->ne[2] * residual->ne[3]; + const int64_t p_tokens = post->ne[1] * post->ne[2] * post->ne[3]; + const int64_t d_tokens = dst->ne[2] * dst->ne[3]; + if (x_tokens != r_tokens || x_tokens != p_tokens || x_tokens != d_tokens) { + return false; + } + if (comb && (comb->ne[2] * comb->ne[3] != x_tokens)) { + return false; + } + + if (x->ne[2] == 1 && x->ne[3] == 1 && + residual->ne[3] == 1 && + post->ne[2] == 1 && post->ne[3] == 1 && + dst->ne[3] == 1 && + (!comb || comb->ne[3] == 1)) { + return false; + } + + if (x->ne[2] > 1 && x->nb[2] != x->ne[1] * x->nb[1]) return false; + if (x->ne[3] > 1 && x->nb[3] != x->ne[2] * x->nb[2]) return false; + + if (residual->ne[3] > 1 && residual->nb[3] != residual->ne[2] * residual->nb[2]) return false; + if (dst->ne[3] > 1 && dst->nb[3] != dst->ne[2] * dst->nb[2]) return false; + + if (post->ne[2] > 1 && post->nb[2] != post->ne[1] * post->nb[1]) return false; + if (post->ne[3] > 1 && post->nb[3] != post->ne[2] * post->nb[2]) return false; + + if (comb && comb->ne[3] > 1 && comb->nb[3] != comb->ne[2] * comb->nb[2]) return false; + + return true; +} + +static void htp_dsv4_hc_post_collapse_tensors( + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct ggml_tensor * x_c, + struct ggml_tensor * r_c, + struct ggml_tensor * p_c, + struct ggml_tensor * c_c, + struct ggml_tensor * d_c +) { + const int64_t n_tokens = x->ne[1] * x->ne[2] * x->ne[3]; + + *x_c = *x; + x_c->ne[1] = n_tokens; + x_c->ne[2] = 1; + x_c->ne[3] = 1; + x_c->nb[2] = x_c->ne[1] * x_c->nb[1]; + x_c->nb[3] = x_c->nb[2]; + + *r_c = *residual; + r_c->ne[2] = n_tokens; + r_c->ne[3] = 1; + r_c->nb[3] = r_c->ne[2] * r_c->nb[2]; + + *d_c = *dst; + d_c->ne[2] = n_tokens; + d_c->ne[3] = 1; + d_c->nb[3] = d_c->ne[2] * d_c->nb[2]; + + *p_c = *post; + p_c->ne[1] = n_tokens; + p_c->ne[2] = 1; + p_c->ne[3] = 1; + p_c->nb[2] = p_c->ne[1] * p_c->nb[1]; + p_c->nb[3] = p_c->nb[2]; + + if (comb && c_c) { + *c_c = *comb; + c_c->ne[2] = n_tokens; + c_c->ne[3] = 1; + c_c->nb[3] = c_c->ne[2] * c_c->nb[2]; + } +} + +static bool htp_kparams_solve_dsv4_hc_post_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * x, + const struct ggml_tensor * residual, + const struct ggml_tensor * post, + const struct ggml_tensor * comb, + const struct ggml_tensor * dst, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + if (x->type != GGML_TYPE_F32 || residual->type != GGML_TYPE_F32 || post->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32) { + return false; + } + if (comb && comb->type != GGML_TYPE_F32) { + return false; + } + if (x->ne[2] != 1 || x->ne[3] != 1) { + return false; + } + if (x->ne[0] % HTP_DSV4_HC_CHUNK_ELEMS != 0) { + return false; + } + if (residual->ne[0] != x->ne[0] || residual->ne[1] != 4 || residual->ne[2] != x->ne[1] || residual->ne[3] != 1) { + return false; + } + if (post->ne[0] != 4 || post->ne[1] != x->ne[1] || post->ne[2] != 1 || post->ne[3] != 1) { + return false; + } + if (comb && (comb->ne[0] != 4 || comb->ne[1] != 4 || comb->ne[2] != x->ne[1] || comb->ne[3] != 1)) { + return false; + } + if (dst->ne[0] != x->ne[0] || dst->ne[1] != 4 || dst->ne[2] != x->ne[1] || dst->ne[3] != 1) { + return false; + } + if (x->nb[0] != sizeof(float) || residual->nb[0] != sizeof(float) || post->nb[0] != sizeof(float) || dst->nb[0] != sizeof(float)) { + return false; + } + if (comb && comb->nb[0] != sizeof(float)) { + return false; + } + if (ggml_is_permuted(x) || ggml_is_permuted(residual) || ggml_is_permuted(post) || ggml_is_permuted(dst) || (comb && ggml_is_permuted(comb))) { + return false; + } + + const uint32_t n_tokens = (uint32_t) x->ne[1]; + + kparams->has_comb = comb != nullptr; + kparams->kernel_type = kparams->has_comb ? HTP_DSV4_HC_POST_KERNEL_COMB : HTP_DSV4_HC_POST_KERNEL_IDENTITY; + kparams->n_chunks = ((uint32_t) x->ne[0] + HTP_DSV4_HC_CHUNK_ELEMS - 1) / HTP_DSV4_HC_CHUNK_ELEMS; + kparams->total_jobs = n_tokens * kparams->n_chunks; + if (kparams->total_jobs == 0) { + return true; + } + kparams->n_threads = (std::min)(htp_session_n_threads(sess), kparams->total_jobs); + kparams->jobs_per_thread = (kparams->total_jobs + kparams->n_threads - 1) / kparams->n_threads; + + const size_t vtcm_budget = htp_session_vtcm_size(sess); + uint32_t block_chunks = (std::min)((uint32_t) HTP_DSV4_HC_MAX_CHUNKS, kparams->n_chunks); + while (block_chunks > 16) { + const uint32_t stride = block_chunks * 128; + const uint32_t buf_sz = HTP_DSV4_HC_POST_ROWS * stride; + if (2 * buf_sz * kparams->n_threads <= vtcm_budget) { + break; + } + block_chunks /= 2; + } + + kparams->dma_block_chunks = block_chunks; + kparams->dma_row_stride = block_chunks * 128; + kparams->dma_buffer_size = HTP_DSV4_HC_POST_ROWS * kparams->dma_row_stride; + kparams->vtcm_size = 2 * kparams->dma_buffer_size * kparams->n_threads; + if (kparams->vtcm_size > vtcm_budget) { + return false; + } + + kparams->div_n_chunks = init_fastdiv_values(kparams->n_chunks); + kparams->div_n_threads = init_fastdiv_values(kparams->n_threads); + + return true; +} + +static inline bool htp_kparams_solve_dsv4_hc_post( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_dsv4_hc_post_kernel_params * kparams +) { + if (htp_dsv4_hc_post_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c); + return htp_kparams_solve_dsv4_hc_post_impl(sess, &x_c, &r_c, &p_c, op->src[3] ? &c_c : nullptr, &d_c, kparams); + } + return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} + template static inline bool htp_kparams_invoke( const struct ggml_hexagon_session * sess, @@ -4252,6 +4630,48 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & } return; } + + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + if (node.inputs.size() < 2 || !node.inputs[0] || !node.inputs[1] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * weights = node.inputs[1]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_pre_can_collapse(x, weights, dst)) { + struct ggml_tensor x_c, w_c, d_c; + htp_dsv4_hc_pre_collapse_tensors(x, weights, dst, &x_c, &w_c, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(w_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } + + if (node.opcode == HTP_OP_DSV4_HC_POST) { + if (node.inputs.size() < 3 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * x = node.inputs[0]; + const struct ggml_tensor * residual = node.inputs[1]; + const struct ggml_tensor * post = node.inputs[2]; + const struct ggml_tensor * comb = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_dsv4_hc_post_can_collapse(x, residual, post, comb, dst)) { + struct ggml_tensor x_c, r_c, p_c, c_c, d_c; + htp_dsv4_hc_post_collapse_tensors(x, residual, post, comb, dst, &x_c, &r_c, &p_c, comb ? &c_c : nullptr, &d_c); + node.inputs[0] = node.add_dummy(x_c); + node.inputs[1] = node.add_dummy(r_c); + node.inputs[2] = node.add_dummy(p_c); + if (comb) { + node.inputs[3] = node.add_dummy(c_c); + } + node.outputs[0] = node.add_dummy(d_c); + } + return; + } } bool htp_kparams_solve( @@ -4284,6 +4704,14 @@ bool htp_kparams_solve( if (node.opcode == HTP_OP_UPSCALE) { return htp_kparams_solve_upscale(sess, src0, dst, node.kparams()); } + if (node.opcode == HTP_OP_DSV4_HC_PRE) { + return htp_kparams_solve_dsv4_hc_pre_impl(sess, src0, src1, dst, node.kparams()); + } + if (node.opcode == HTP_OP_DSV4_HC_POST) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams()); + } return htp_kparams_solve(sess, node.node, node.kernel_params); } @@ -4391,6 +4819,12 @@ bool htp_kparams_solve( case HTP_OP_SOLVE_TRI: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_DSV4_HC_PRE: + return htp_kparams_invoke(sess, op, kparams); + + case HTP_OP_DSV4_HC_POST: + return htp_kparams_invoke(sess, op, kparams); + default: if (htp_op_is_unary(opcode)) { return htp_kparams_invoke(sess, op, kparams); diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h index b757f430b83c..9f2896eb507e 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.h +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -11,6 +11,7 @@ #include "htp/cpy-ops.h" #include "htp/cumsum-ops.h" #include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" #include "htp/fill-ops.h" #include "htp/flash-attn-ops.h" #include "htp/gated-delta-net-ops.h" diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp index d4e5150c0f28..36e7a3f38c2f 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.cpp +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -5,6 +5,7 @@ #include "htp/binary-ops.h" #include "htp/cumsum-ops.h" #include "htp/diag-ops.h" +#include "htp/dsv4-hc-ops.h" #include "htp/fill-ops.h" #include "htp/flash-attn-ops.h" #include "htp/gated-delta-net-ops.h" @@ -185,6 +186,16 @@ static std::string format_kernel_params(const htp_opnode & node) { const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_PRE) { + const auto * kparams = node.kparams(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_PRE_KERNEL_GATED ? "gated" : "plain"; + snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str, + (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_DSV4_HC_POST) { + const auto * kparams = node.kparams(); + const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity"; + snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str, + (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { const auto * kparams = node.kparams(); diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index b5d6d55daf41..7c7c5be17ce1 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -49,6 +49,7 @@ add_library(${HTP_LIB} SHARED pool-ops.c upscale-ops.c allreduce-ops.c + dsv4-hc-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c new file mode 100644 index 000000000000..62f12c55d217 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.c @@ -0,0 +1,675 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" + +#include + +#include +#include +#include + +#include "dma-queue.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "htp-tensor.h" +#include "hvx-utils.h" +#include "dsv4-hc-ops.h" + +// --------------------------------------------------------------------------- +// DSV4_HC_PRE +// --------------------------------------------------------------------------- + +// hvx_vec_fast_sigmoid_f32_guard_2it with fewer live constants and predicates +static inline HVX_Vector dsv4_hc_sigmoid_f32(HVX_Vector v, HVX_Vector max_exp, HVX_Vector min_exp) { + v = Q6_Vsf_vmin_VsfVsf(Q6_Vsf_vmax_VsfVsf(v, min_exp), max_exp); + v = Q6_Vqf32_vmpy_VsfVsf(v, hvx_vec_splat_f32(0.5f * 1.442695022f)); + + HVX_Vector in_int = Q6_Vw_equals_Vsf(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector r = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + for (int i = 0; i < 2; ++i) { + const HVX_Vector rv = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(r, v5)); + r = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(Q6_Vqf32_vadd_VsfVsf(r, r), Q6_Vqf32_vmpy_VsfVsf(r, rv))); + } + + return Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(v3, r)); +} + +struct dsv4_hc_pre_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_pre_kernel_params * kparams; + uint32_t token_start; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_thread; + float scale; + uint8_t * vtcm; + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; +}; + +struct dsv4_hc_block { + uint32_t it; + uint32_t chunk; + uint32_t count; +}; + +static inline struct dsv4_hc_block dsv4_hc_pre_block_make( + const struct dsv4_hc_pre_context * dctx, + uint32_t job, + uint32_t end) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->n_chunks; + const uint32_t count = hex_smin(dctx->dma_block_chunks, hex_smin(dctx->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static inline void dsv4_hc_pre_push_plain( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * weights = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 5*row_stride, weights->data + (block->it*weights->nb[1])), 16, 16, 16, 1); +} + +static inline void dsv4_hc_pre_push_gated( + dma_queue * dma_q, + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * g = dctx->octx->src[1]; + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, x->data + (block->it*x->nb[2] + block->chunk*VLEN)), row_stride, x->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, g->data + (block->it*g->nb[2] + block->chunk*VLEN)), row_stride, g->nb[1], row_size, 4); +} + +static __attribute__((noinline)) void dsv4_hc_pre_plain_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + uint8_t * d_spad, + const float * wb) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector w0 = hvx_vec_splat_f32(wb[0]); + const HVX_Vector w1 = hvx_vec_splat_f32(wb[1]); + const HVX_Vector w2 = hvx_vec_splat_f32(wb[2]); + const HVX_Vector w3 = hvx_vec_splat_f32(wb[3]); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + + HVX_Vector x0 = *(const HVX_Vector *) xb; + HVX_Vector x1 = *(const HVX_Vector *) (xb + row_stride); + HVX_Vector x2 = *(const HVX_Vector *) (xb + 2*row_stride); + HVX_Vector x3 = *(const HVX_Vector *) (xb + 3*row_stride); + + HVX_Vector acc = hvx_vec_mul_f32_f32(x0, w0); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x1, w1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x2, w2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(x3, w3)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static __attribute__((noinline)) void dsv4_hc_pre_gated_block_f32( + const struct dsv4_hc_pre_context * dctx, + const struct dsv4_hc_block * block, + const uint8_t * x_spad, + const uint8_t * g_spad, + uint8_t * d_spad) { + const uint32_t row_stride = dctx->dma_row_stride; + const HVX_Vector scale_v = hvx_vec_splat_f32(dctx->scale); + const HVX_Vector max_exp = hvx_vec_splat_f32(87.0f); + const HVX_Vector min_exp = hvx_vec_splat_f32(-87.0f); + + for (uint32_t c = 0; c < block->count; ++c) { + const uint8_t * xb = x_spad + c*VLEN; + const uint8_t * gb = g_spad + c*VLEN; + + HVX_Vector xv = *(const HVX_Vector *) xb; + HVX_Vector gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) gb, max_exp, min_exp); + HVX_Vector acc = hvx_vec_mul_f32_f32(xv, gv); + + xv = *(const HVX_Vector *) (xb + row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 2*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 2*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + xv = *(const HVX_Vector *) (xb + 3*row_stride); + gv = dsv4_hc_sigmoid_f32(*(const HVX_Vector *) (gb + 3*row_stride), max_exp, min_exp); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(xv, gv)); + + *(HVX_Vector *) (d_spad + c*VLEN) = hvx_vec_mul_f32_f32(acc, scale_v); + } +} + +static void dsv4_hc_pre_thread_plain_f32(unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_thread * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, weights in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 4*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_plain(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * wb = (const float *) (spad[curr] + 5*row_stride); + uint8_t * d_spad = spad[curr] + 4*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_plain_block_f32(dctx, &block, spad[curr], d_spad, wb); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_plain(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_pre_thread_gated_f32(unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_pre_context * dctx = (struct dsv4_hc_pre_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t row_stride = dctx->dma_row_stride; + const uint32_t buffer_size = dctx->dma_buffer_size; + const uint32_t start = dctx->jobs_per_thread * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + // empty dst writeback keeps pop order dst, x, g in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 8*row_stride), block.count*VLEN, block.count*VLEN, 0, 0); + dsv4_hc_pre_push_gated(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_pre_block_make(dctx, job, end); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + uint8_t * d_spad = spad[curr] + 8*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_pre_gated_block_f32(dctx, &block, spad[curr], spad[curr] + 4*row_stride, d_spad); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[1] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), block.count*VLEN, block.count*VLEN, block.count*VLEN, 1); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_pre_block_make(dctx, prefetch_job, end); + dsv4_hc_pre_push_gated(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_pre(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * weights = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_pre_kernel_params * kparams = + (const struct htp_dsv4_hc_pre_kernel_params *) octx->kernel_params; + + const uint32_t n_threads = kparams->n_threads; + if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[2]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_thread = kparams->jobs_per_thread; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[1] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + float scale; + memcpy(&scale, octx->op_params, sizeof(scale)); + + struct dsv4_hc_pre_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .n_chunks = kparams->n_chunks, + .total_jobs = total_jobs, + .jobs_per_thread = jobs_per_thread, + .scale = scale, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + .dma_block_chunks = kparams->dma_block_chunks, + .dma_row_stride = kparams->dma_row_stride, + .dma_buffer_size = kparams->dma_buffer_size, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_PRE_KERNEL_PLAIN: worker = dsv4_hc_pre_thread_plain_f32; break; + case HTP_DSV4_HC_PRE_KERNEL_GATED: worker = dsv4_hc_pre_thread_gated_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-pre: (%ux%ux%u) -> (%ux%u) : ktype %u tokens %u:%u nth %u\n", + x->ne[0], x->ne[1], x->ne[2], dst->ne[0], dst->ne[1], kparams->kernel_type, + token_start, token_start + n_tokens, n_threads); + + work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads); + return HTP_STATUS_OK; +} + +// --------------------------------------------------------------------------- +// DSV4_HC_POST +// --------------------------------------------------------------------------- + +struct dsv4_hc_post_context { + struct htp_ops_context * octx; + const struct htp_dsv4_hc_post_kernel_params * kparams; + uint32_t token_start; + uint32_t total_jobs; + uint32_t jobs_per_thread; + uint8_t * vtcm; +}; + +static inline struct dsv4_hc_block dsv4_hc_post_block_make( + const struct dsv4_hc_post_context * dctx, + uint32_t job, + uint32_t end, + uint32_t block_chunks) { + const uint32_t local_token = fastdiv(job, &dctx->kparams->div_n_chunks); + const uint32_t chunk = job - local_token*dctx->kparams->n_chunks; + const uint32_t count = hex_smin(block_chunks, hex_smin(dctx->kparams->n_chunks - chunk, end - job)); + return (struct dsv4_hc_block) { + .it = dctx->token_start + local_token, + .chunk = chunk, + .count = count, + }; +} + +static __attribute__((noinline)) void dsv4_hc_post_identity_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const uint8_t * rr = rb + id*r_stride; + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const HVX_Vector xv = *(const HVX_Vector *) (xb + i*VLEN); + const HVX_Vector rv = *(const HVX_Vector *) (rr + i*VLEN); + *(HVX_Vector *) (dd + i*VLEN) = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(xv, pv), rv); + } + } +} + +static __attribute__((noinline)) void dsv4_hc_post_comb_block_f32( + const uint8_t * xb, + const uint8_t * rb, + size_t r_stride, + uint8_t * db, + size_t d_stride, + uint32_t count, + const float * p, + const float * c) { + for (uint32_t id = 0; id < 4; ++id) { + const HVX_Vector pv = hvx_vec_splat_f32(p[id]); + const HVX_Vector c0 = hvx_vec_splat_f32(c[id]); + const HVX_Vector c1 = hvx_vec_splat_f32(c[4 + id]); + const HVX_Vector c2 = hvx_vec_splat_f32(c[8 + id]); + const HVX_Vector c3 = hvx_vec_splat_f32(c[12 + id]); + uint8_t * dd = db + id*d_stride; + + for (uint32_t i = 0; i < count; ++i) { + const uint8_t * r = rb + i*VLEN; + HVX_Vector acc = hvx_vec_mul_f32_f32(*(const HVX_Vector *) (xb + i*VLEN), pv); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) r, c0)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + r_stride), c1)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 2*r_stride), c2)); + acc = hvx_vec_add_f32_f32(acc, hvx_vec_mul_f32_f32(*(const HVX_Vector *) (r + 3*r_stride), c3)); + *(HVX_Vector *) (dd + i*VLEN) = acc; + } + } +} + +static inline void dsv4_hc_post_push_identity( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * x = dctx->octx->src[0]; + const struct htp_tensor * residual = dctx->octx->src[1]; + const struct htp_tensor * post = dctx->octx->src[2]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t row_size = block->count * VLEN; + + dma_queue_push(dma_q, dma_make_data(spad, residual->data + (block->it*residual->nb[2] + block->chunk*VLEN)), + row_stride, residual->nb[1], row_size, 4); + dma_queue_push(dma_q, dma_make_data(spad + 4*row_stride, x->data + (block->it*x->nb[1] + block->chunk*VLEN)), + row_stride, row_size, row_size, 1); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride, post->data + (block->it*post->nb[1])), + 16, 16, 16, 1); +} + +static inline void dsv4_hc_post_push_comb( + dma_queue * dma_q, + const struct dsv4_hc_post_context * dctx, + const struct dsv4_hc_block * block, + uint8_t * spad) { + const struct htp_tensor * comb = dctx->octx->src[3]; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + + dsv4_hc_post_push_identity(dma_q, dctx, block, spad); + dma_queue_push(dma_q, dma_make_data(spad + 9*row_stride + 64, comb->data + (block->it*comb->nb[2])), + 16, comb->nb[1], 16, 4); +} + +static void dsv4_hc_post_thread_identity_f32(unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_thread * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_identity(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_identity_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_identity(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +static void dsv4_hc_post_thread_comb_f32(unsigned int nth, unsigned int ith, void * data) { + struct dsv4_hc_post_context * dctx = (struct dsv4_hc_post_context *) data; + struct htp_ops_context * octx = dctx->octx; + + const struct htp_tensor * dst = octx->dst; + const uint64_t d_data = dst->data; + + const uint32_t block_chunks = dctx->kparams->dma_block_chunks; + const uint32_t row_stride = dctx->kparams->dma_row_stride; + const uint32_t buffer_size = dctx->kparams->dma_buffer_size; + const uint32_t start = dctx->jobs_per_thread * ith; + const uint32_t end = hex_smin(start + dctx->jobs_per_thread, dctx->total_jobs); + if (start >= end) { + return; + } + + uint8_t * spad[2] = { + dctx->vtcm + 2*buffer_size*ith, + dctx->vtcm + 2*buffer_size*ith + buffer_size, + }; + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint32_t prefetch_job = start; + for (uint32_t i = 0; i < 2 && prefetch_job < end; ++i) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + // empty dst writeback keeps pop order dst, residual, x, post, comb in loop + dma_queue_push(dma_q, dma_make_data(d_data, spad[i] + 5*row_stride), dst->nb[1], row_stride, 0, 0); + dsv4_hc_post_push_comb(dma_q, dctx, &block, spad[i]); + prefetch_job += block.count; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + uint32_t job = start; + uint32_t curr = 0; + while (job < end) { + const struct dsv4_hc_block block = dsv4_hc_post_block_make(dctx, job, end, block_chunks); + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + const float * p = (const float *) (spad[curr] + 9*row_stride); + const float * c = (const float *) (spad[curr] + 9*row_stride + 64); + uint8_t * d_spad = spad[curr] + 5*row_stride; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + dsv4_hc_post_comb_block_f32(spad[curr] + 4*row_stride, spad[curr], row_stride, d_spad, row_stride, block.count, p, c); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) block.it); + + const uint64_t d_dst = d_data + (block.it*dst->nb[2] + block.chunk*VLEN); + dma_queue_push(dma_q, dma_make_data(d_dst, d_spad), dst->nb[1], row_stride, block.count*VLEN, 4); + + if (prefetch_job < end) { + const struct dsv4_hc_block next = dsv4_hc_post_block_make(dctx, prefetch_job, end, block_chunks); + dsv4_hc_post_push_comb(dma_q, dctx, &next, spad[curr]); + prefetch_job += next.count; + } + + job += block.count; + curr ^= 1; + } + + dma_queue_flush(dma_q); + + (void) nth; +} + +int op_dsv4_hc_post(struct htp_ops_context * octx) { + const struct htp_tensor * x = octx->src[0]; + const struct htp_tensor * residual = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + const struct htp_dsv4_hc_post_kernel_params * kparams = + (const struct htp_dsv4_hc_post_kernel_params *) octx->kernel_params; + + const uint32_t n_threads = kparams->n_threads; + if (!htp_ops_context_set_n_threads_div(octx, n_threads, kparams->div_n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + uint32_t token_start = 0; + uint32_t n_tokens = x->ne[1]; + uint32_t total_jobs = kparams->total_jobs; + uint32_t jobs_per_thread = kparams->jobs_per_thread; + if (octx->ctx->mdev.count > 1) { + const bool can_split = htp_tensor_mdev_data_aligned(dst) && ((dst->nb[2] & (HTP_TENSOR_MDEV_LINE_SIZE - 1)) == 0); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(n_tokens, can_split ? 1 : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + token_start = range.start; + n_tokens = range.count; + total_jobs = n_tokens * kparams->n_chunks; + jobs_per_thread = fastdiv(total_jobs + n_threads - 1, &kparams->div_n_threads); + } + + if (n_tokens == 0) { + return HTP_STATUS_OK; + } + + struct dsv4_hc_post_context dctx = { + .octx = octx, + .kparams = kparams, + .token_start = token_start, + .total_jobs = total_jobs, + .jobs_per_thread = jobs_per_thread, + .vtcm = (uint8_t *) octx->ctx->vtcm_base, + }; + + work_queue_func_t worker = NULL; + switch (kparams->kernel_type) { + case HTP_DSV4_HC_POST_KERNEL_IDENTITY: worker = dsv4_hc_post_thread_identity_f32; break; + case HTP_DSV4_HC_POST_KERNEL_COMB: worker = dsv4_hc_post_thread_comb_f32; break; + default: return HTP_STATUS_INVAL_PARAMS; + } + + FARF(HIGH, "dsv4-hc-post: (%ux%u) (%ux%ux%u) -> (%ux%ux%u) : ktype %u tokens %u:%u nth %u\n", + x->ne[0], x->ne[1], residual->ne[0], residual->ne[1], residual->ne[2], + dst->ne[0], dst->ne[1], dst->ne[2], kparams->kernel_type, + token_start, token_start + n_tokens, n_threads); + + work_queue_run(octx->ctx->work_queue, worker, &dctx, n_threads); + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h new file mode 100644 index 000000000000..a0c01aa91fd1 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/dsv4-hc-ops.h @@ -0,0 +1,77 @@ +#ifndef HTP_DSV4_HC_OPS_H +#define HTP_DSV4_HC_OPS_H + +#include + +#include "hex-fastdiv.h" +#include "htp-ops.h" + +#define HTP_DSV4_HC_CHUNK_ELEMS 32 +#define HTP_DSV4_HC_MAX_CHUNKS 128 +#define HTP_DSV4_HC_PRE_GATED_ROWS 9 +#define HTP_DSV4_HC_PRE_PLAIN_ROWS 6 + +enum htp_dsv4_hc_pre_kernel_type { + HTP_DSV4_HC_PRE_KERNEL_PLAIN = 0, + HTP_DSV4_HC_PRE_KERNEL_GATED = 1, +}; + +struct htp_dsv4_hc_pre_kernel_params { + uint32_t kernel_type; + uint32_t n_threads; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_thread; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_threads; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_pre_kernel_params) <= 128, "htp_dsv4_hc_pre_kernel_params is too large for kernel_params blob"); +#endif + +// dma buffer holds residual rows 0..3, x at row 4, dst rows 5..8, post/comb at row 9 +#define HTP_DSV4_HC_POST_ROWS 10 + +enum htp_dsv4_hc_post_kernel_type { + HTP_DSV4_HC_POST_KERNEL_IDENTITY = 0, + HTP_DSV4_HC_POST_KERNEL_COMB = 1, +}; + +struct htp_dsv4_hc_post_kernel_params { + uint32_t kernel_type; + uint32_t n_threads; + uint32_t n_chunks; + uint32_t total_jobs; + uint32_t jobs_per_thread; + uint32_t has_comb; + + uint32_t dma_block_chunks; + uint32_t dma_row_stride; + uint32_t dma_buffer_size; + uint32_t vtcm_size; + + struct fastdiv_values div_n_chunks; + struct fastdiv_values div_n_threads; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_dsv4_hc_post_kernel_params) <= 128, "htp_dsv4_hc_post_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_ops_context; + +int op_dsv4_hc_pre(struct htp_ops_context * octx); +int op_dsv4_hc_post(struct htp_ops_context * octx); + +#endif // HTP_DSV4_HC_OPS_H diff --git a/ggml/src/ggml-hexagon/htp/glu-ops.c b/ggml/src/ggml-hexagon/htp/glu-ops.c index b4edddf3fe0a..dc621bf46c10 100644 --- a/ggml/src/ggml-hexagon/htp/glu-ops.c +++ b/ggml/src/ggml-hexagon/htp/glu-ops.c @@ -186,53 +186,6 @@ static void swiglu_clamp_f32(const float * restrict src0, static const float GELU_COEF_A = 0.044715f; static const float SQRT_2_OVER_PI = 0.79788456080286535587989211986876f; -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { - v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); - - HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); - HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); - HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); - - HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); - v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); - - HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); - v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); - v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); - - HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); - v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); - - HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); - HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); - - // Newton-Raphson with 2 iterations - HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); - HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); - HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( - i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); - r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( - r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); - HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); - - res = Q6_Vqf32_vmpy_VsfVsf(v3, res); - - return Q6_Vsf_equals_Vqf32(res); -} - -static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, - HVX_Vector one, - HVX_Vector max_exp, - HVX_Vector min_exp) { - const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); - const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); - - HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); - out = Q6_V_vmux_QVV(pred_max, out, one); - return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); -} - static inline void hvx_geglu_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { assert((unsigned long) dst % 128 == 0); assert((unsigned long) src0 % 128 == 0); diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 0c455141d384..4e2386bfa1ab 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -131,21 +131,34 @@ static inline void htp_ops_context_reset(struct htp_ops_context * octx, struct h octx->n_threads_div = ctx->n_threads_div; } -static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { +static inline bool htp_ops_context_set_n_threads_div( + struct htp_ops_context * octx, + uint32_t n_threads, + struct fastdiv_values n_threads_div) { if (n_threads == 0 || n_threads > octx->ctx->n_threads) { return false; } - if (n_threads != octx->n_threads) { - octx->n_threads = n_threads; - octx->n_threads_div = n_threads == octx->ctx->n_threads - ? octx->ctx->n_threads_div - : init_fastdiv_values(n_threads); - } + octx->n_threads = n_threads; + octx->n_threads_div = n_threads_div; return true; } +static inline bool htp_ops_context_set_n_threads(struct htp_ops_context * octx, uint32_t n_threads) { + if (n_threads == 0 || n_threads > octx->ctx->n_threads) { + return false; + } + if (n_threads == octx->n_threads) { + return true; + } + + const struct fastdiv_values n_threads_div = n_threads == octx->ctx->n_threads + ? octx->ctx->n_threads_div + : init_fastdiv_values(n_threads); + return htp_ops_context_set_n_threads_div(octx, n_threads, n_threads_div); +} + static inline void htp_ops_context_set_status(struct htp_ops_context * octx, int status) { if (status > HTP_STATUS_OK && octx->status == HTP_STATUS_OK) { octx->status = status; diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index b8df01830b84..1e132f877952 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -117,6 +117,8 @@ enum htp_op_code { HTP_OP_UNARY_CEIL, HTP_OP_UNARY_SGN, HTP_OP_UPSCALE, + HTP_OP_DSV4_HC_PRE, + HTP_OP_DSV4_HC_POST, HTP_OP_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index d4d67ff2f07d..3f253a0b0325 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -52,6 +52,53 @@ static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard(HVX_Vector v, return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); } +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_2it(HVX_Vector v) { + v = Q6_Vqf32_vmpy_VsfVsf(v, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + v = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(v), Q6_V_vsplat_R(FAST_SIGMOID_C3)); + + HVX_Vector in_int = hvx_vec_truncate_f32(Q6_Vsf_equals_Vqf32(v)); + HVX_Vector x = Q6_Vqf32_vsub_Vqf32Vsf(v, Q6_Vsf_equals_Vw(in_int)); + HVX_Vector xx = Q6_Vqf32_vmpy_Vqf32Vqf32(x, x); + + HVX_Vector v1 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(xx), Q6_V_vsplat_R(FAST_SIGMOID_C2)); + v1 = Q6_Vqf32_vadd_Vqf32Vsf(v1, Q6_V_vsplat_R(FAST_SIGMOID_LOG2F)); + + HVX_Vector v2 = Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(x), Q6_V_vsplat_R(FAST_SIGMOID_C1)); + v2 = Q6_Vqf32_vmpy_Vqf32Vqf32(v2, xx); + v2 = Q6_Vqf32_vadd_Vqf32Vqf32(v2, x); + + HVX_Vector v3 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(v2, v1)); + v3 = Q6_Vw_vaslacc_VwVwR(v3, in_int, 24); + + HVX_Vector v4 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(v2, v1)); + HVX_Vector v5 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(v3, v4)); + + // Newton-Raphson with 2 iterations + HVX_Vector two_sf = hvx_vec_splat_f32(2.0f); + HVX_Vector i_sf = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(0x7EEEEBB3), v5); + HVX_Vector r_qf = Q6_Vqf32_vmpy_VsfVsf( + i_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(i_sf, v5))))); + r_qf = Q6_Vqf32_vmpy_Vqf32Vqf32( + r_qf, Q6_Vqf32_vsub_VsfVsf(two_sf, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(Q6_Vsf_equals_Vqf32(r_qf), v5)))); + HVX_Vector res = Q6_Vsf_equals_Vqf32(r_qf); + + res = Q6_Vqf32_vmpy_VsfVsf(v3, res); + + return Q6_Vsf_equals_Vqf32(res); +} + +static inline HVX_Vector hvx_vec_fast_sigmoid_f32_guard_2it(HVX_Vector v, + HVX_Vector one, + HVX_Vector max_exp, + HVX_Vector min_exp) { + const HVX_VectorPred pred_max = Q6_Q_vcmp_gt_VsfVsf(max_exp, v); + const HVX_VectorPred pred_min = Q6_Q_vcmp_gt_VsfVsf(v, min_exp); + + HVX_Vector out = hvx_vec_fast_sigmoid_f32_2it(v); + out = Q6_V_vmux_QVV(pred_max, out, one); + return Q6_V_vmux_QVV(pred_min, out, Q6_V_vzero()); +} + static inline HVX_Vector hvx_vec_tanh_f32(HVX_Vector x) { // tanh(x) = 2 * sigmoid(2x) - 1 HVX_Vector two = hvx_vec_splat_f32(2.0f); diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 78adc9a495d5..4c82c684dc66 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -34,6 +34,7 @@ #include "work-queue.h" #include "hex-profile.h" #include "allreduce-ops.h" +#include "dsv4-hc-ops.h" #include "htp-fence.h" #define HMX_QUEUE_CAPACITY 128 @@ -1040,6 +1041,12 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_POOL_1D: return op_pool_1d(octx); + case HTP_OP_DSV4_HC_PRE: + return op_dsv4_hc_pre(octx); + + case HTP_OP_DSV4_HC_POST: + return op_dsv4_hc_post(octx); + case HTP_OP_CONCAT: return op_concat(octx); diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index 99b43bb2b347..bef6d4e50bcd 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -177,11 +177,17 @@ def device_arg_from_devices(devices): def normalize_cmd_device_args(cmd_args): + exec_name = os.path.basename(cmd_args[0]) if cmd_args else "" + if exec_name.endswith(".exe"): + exec_name = exec_name[:-4] + for i, arg in enumerate(cmd_args): if arg == "--device" and i + 1 < len(cmd_args): cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) elif arg.startswith("--device="): cmd_args[i] = "--device=" + device_arg_from_devices(arg.split("=", 1)[1]) + elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args): + cmd_args[i + 1] = device_arg_from_devices(cmd_args[i + 1]) def main(): @@ -299,6 +305,10 @@ def set_env(env_name, opt_val): env_vars[env_name] = str(opt_val) # Resolve and filter devices (HTP vs OpenCL) + exec_name = os.path.basename(cmd_args[0]) if cmd_args else "" + if exec_name.endswith(".exe"): + exec_name = exec_name[:-4] + device_in_cmd = None for i, arg in enumerate(cmd_args): if arg == "--device" and i + 1 < len(cmd_args): @@ -307,6 +317,9 @@ def set_env(env_name, opt_val): elif arg.startswith("--device="): device_in_cmd = arg.split("=", 1)[1] break + elif exec_name in ("test-backend-ops", "test-llama-archs") and arg in ("-b", "--backend") and i + 1 < len(cmd_args): + device_in_cmd = cmd_args[i + 1] + break if args.devices is not None: devices_val = args.devices @@ -402,7 +415,7 @@ def set_env(env_name, opt_val): # Resolve executable path executable = cmd_args[0] - known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops"] + known_binaries = ["llama-cli", "llama-bench", "llama-completion", "llama-mtmd-cli", "test-backend-ops", "test-llama-archs"] if executable in known_binaries: if target_type in ("android", "linux"): resolved_exec = f"./bin/{executable}" @@ -419,22 +432,23 @@ def set_env(env_name, opt_val): basename = basename[:-4] device_val = None - if basename == "test-backend-ops": - for i in range(len(cmd_args)): - if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args): - val = cmd_args[i + 1] - new_val = "" - for j, char in enumerate(val): - if char in ('[', ']'): - if j > 0 and val[j - 1] == '\\': - new_val += char + if basename in ("test-backend-ops", "test-llama-archs"): + if basename == "test-backend-ops": + for i in range(len(cmd_args)): + if cmd_args[i] in ("-p", "--params") and i + 1 < len(cmd_args): + val = cmd_args[i + 1] + new_val = "" + for j, char in enumerate(val): + if char in ('[', ']'): + if j > 0 and val[j - 1] == '\\': + new_val += char + else: + new_val += '\\' + char else: - new_val += '\\' + char - else: - new_val += char - cmd_args[i + 1] = new_val + new_val += char + cmd_args[i + 1] = new_val - has_b = any(arg == "-b" for arg in cmd_args) + has_b = any(arg in ("-b", "--backend") for arg in cmd_args) if not has_b: if args.devices: device_val = device_arg_from_devices(args.devices) @@ -462,9 +476,13 @@ def set_env(env_name, opt_val): or args.hex_profile is not None or args.hex_optrace is not None ) - if verbose_trigger and basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"): - if "-v" not in cmd_args and "--verbose" not in cmd_args: - cmd_args.append("-v") + if verbose_trigger: + if basename == "test-llama-archs": + if "-v" not in cmd_args: + cmd_args += ["-v", "5"] + elif basename in ("llama-cli", "llama-completion", "llama-bench", "llama-server", "llama-mtmd-cli"): + if "-v" not in cmd_args and "--verbose" not in cmd_args: + cmd_args.append("-v") # Inject defaults for llama-cli, llama-completion, and llama-server if not overridden by the user if basename in ("llama-cli", "llama-completion", "llama-server"): diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 68b25c876484..ae910f294b4e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10234,6 +10234,8 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_dsv4_hc_post(31, 17, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(2560, 21, true, true)); test_cases.emplace_back(new test_dsv4_hc_post(31, 17, false, true)); + test_cases.emplace_back(new test_dsv4_hc_post(4096, 1)); // DeepSeek-V4-Flash decode + test_cases.emplace_back(new test_dsv4_hc_post(2560, 1, true, true)); // Qwen4-Exp decode for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { // blocks with extra cells and scores (k-pool) @@ -12771,6 +12773,21 @@ static std::vector> make_test_cases_perf() { // delta-net conv input: conv state followed by the transposed qkv of the ubatch test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {3, 8192, 1, 1}, 512, 0, 16)); + // DSV4_HC_PRE: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_pre(4096, 4, n_tokens, true)); // synthetic, no gated model uses this width + test_cases.emplace_back(new test_dsv4_hc_pre(2560, 4, n_tokens, true)); // Qwen4-Exp (Qwen3.8-Flash-Next) + } + + // DSV4_HC_POST: n_tokens 1 is decode, 128 and 512 are prompt batches + for (int64_t n_tokens : {1, 128, 512}) { + test_cases.emplace_back(new test_dsv4_hc_post(4096, n_tokens)); // DeepSeek-V4-Flash + test_cases.emplace_back(new test_dsv4_hc_post(7168, n_tokens)); // DeepSeek-V4-Pro + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true, true)); // Qwen4-Exp, with the post gate chain + test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain + } + // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { From d1c4e5bec1db1aa7feb2ee62a290e16d14b5c85a Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Tue, 3 Nov 2026 19:02:54 +0100 Subject: [PATCH 031/286] ui: run local browser tests in installed Chrome with its sandbox on (#31596) Playwright's bundled Chromium lags behind Chrome security releases (1.56.1 ships 141, and even 1.64.1 ships 156.0.8078.4, before the 156.0.8078.73 fixes), and Playwright launches it with --no-sandbox by default. Outside CI, use the installed, auto-updated Chrome with the sandbox enabled. CI keeps the bundled Chromium, so the self-hosted runner's preinstalled 1.56.1 browsers still work. Assisted-by: Claude (Opus 5.5 Medium) --- tools/ui/README.md | 2 ++ tools/ui/playwright.config.ts | 7 ++++++- tools/ui/vite.config.ts | 8 +++++--- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/tools/ui/README.md b/tools/ui/README.md index c6ec54721c2a..5db75128cc77 100644 --- a/tools/ui/README.md +++ b/tools/ui/README.md @@ -710,6 +710,8 @@ npm run test:server # Server-side unit tests npm run test:ui # Storybook visual tests ``` +Locally, the browser tests (e2e, client and Storybook) run in your installed Google Chrome with its sandbox on, since Chrome updates itself and Playwright's bundled Chromium lags behind security fixes. CI (`CI` set) uses the bundled Chromium from `npx playwright install`. + ### Storybook Development ```bash diff --git a/tools/ui/playwright.config.ts b/tools/ui/playwright.config.ts index 057ed416df59..2b6dde6cc4c1 100644 --- a/tools/ui/playwright.config.ts +++ b/tools/ui/playwright.config.ts @@ -9,7 +9,12 @@ export default defineConfig({ projects: [ { name: 'chromium', - use: { ...devices['Desktop Chrome'] } + use: { + ...devices['Desktop Chrome'], + // locally, the installed (auto-updated) Chrome with its sandbox on: the bundled + // Chromium lags behind security fixes and Playwright disables its sandbox by default + ...(process.env.CI ? {} : { channel: 'chrome', launchOptions: { chromiumSandbox: true } }) + } } ], reporter: 'line', diff --git a/tools/ui/vite.config.ts b/tools/ui/vite.config.ts index 233ba7452cfe..b213ec8ab4ae 100644 --- a/tools/ui/vite.config.ts +++ b/tools/ui/vite.config.ts @@ -17,10 +17,12 @@ const __dirname = dirname(fileURLToPath(import.meta.url)); const browserBaseConfig: any = { enabled: true, instances: [{ browser: 'chromium' }], + // CI runs the bundled Chromium without its sandbox, which the runners can't start; + // locally, the installed (auto-updated) Chrome with its sandbox on, see playwright.config.ts provider: playwright({ - launchOptions: { - args: ['--no-sandbox'] - } + launchOptions: process.env.CI + ? { args: ['--no-sandbox'] } + : { channel: 'chrome', chromiumSandbox: true } }) }; From 6b9c0cfcba5ef1e86c14f0d4925578e245274d95 Mon Sep 17 00:00:00 2001 From: Zack Li <39573601+zhiyuan8@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:06:28 -0800 Subject: [PATCH 032/286] hexagon: faster TOP_K for the backend sampler (#31279) * hexagon: faster TOP_K for the backend sampler (skip tiles, then block scan) Skip the bitonic sort for tiles whose max cannot reach the running k-th best value, then replace the tile sort with top_k_scan(): reject 256-float blocks with one HVX max-reduce and insert only surviving elements into a sorted k-entry list. Used for k in 2..64 and rows wider than 1024, in both the single-row and multi-row paths. Ties keep the first-seen index. NaN is treated as -INF. * hex-top-k: eliminate scalar reads from vtcm --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/htp/argsort-ops.c | 112 +++++++++++++----------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/argsort-ops.c b/ggml/src/ggml-hexagon/htp/argsort-ops.c index 7aa7ca8e41c9..9351a73f24ab 100644 --- a/ggml/src/ggml-hexagon/htp/argsort-ops.c +++ b/ggml/src/ggml-hexagon/htp/argsort-ops.c @@ -315,55 +315,61 @@ static void bitonic_sort_vtcm_desc(uint8_t * values, uint8_t * indices, uint32_t } } -static void top_k_select_tiled(dma_queue * dma_q, struct htp_thread_trace * tr, - dma_addr_t src_addr, uint32_t n, uint32_t k, - float * values_buf, int32_t * indices_buf, - float * out_values, int32_t * out_indices, - uint32_t chunk_base) { - const uint32_t tile_elems = 1024; - uint32_t n_tiles = (n + tile_elems - 1) / tile_elems; - uint32_t candidate_count = n_tiles * k; - uint32_t merge_n_vec = hmx_ceil_div(candidate_count, 32); - uint32_t merge_n_vec_pow2 = 1; - while (merge_n_vec_pow2 < merge_n_vec) merge_n_vec_pow2 <<= 1; - uint32_t merge_elems = merge_n_vec_pow2 * 32; - float * candidate_values = values_buf + tile_elems; - int32_t * candidate_indices = indices_buf + tile_elems; - uint32_t candidate_pos = 0; - - for (uint32_t offset = 0; offset < n; offset += tile_elems) { - uint32_t tile_count = MIN(tile_elems, n - offset); - dma_sync_read(dma_q, values_buf, src_addr + (size_t) offset * sizeof(float), tile_count * sizeof(float)); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); - if (tile_count < tile_elems) { - hvx_splat_f32_u((uint8_t *) (values_buf + tile_count), -INFINITY, tile_elems - tile_count); +// top-k of one row held in VTCM, k <= 64. Keeps a sorted list of the k best, rejects 256-float blocks with one max-reduce. +// Writes k values and k indices (ties: first seen wins). Slots with no finite value get -INF and index 0. +static void top_k_scan(const float * vals, uint32_t n, uint32_t k, float * out_v, int32_t * out_i) { + float best_v[64]; + int32_t best_i[64]; + uint32_t cnt = 0; + float thr = -INFINITY; + + const uint32_t blk = 256; + for (uint32_t off = 0; off < n; off += blk) { + const uint32_t bn = MIN(blk, n - off); + if (hvx_reduce_max_f32((const uint8_t *) (vals + off), bn) <= thr) { + continue; } - - bitonic_sort_vtcm_desc((uint8_t *) values_buf, (uint8_t *) indices_buf, tile_elems / 32, true); - uint32_t tile_k = MIN(k, tile_count); - for (uint32_t j = 0; j < tile_k; j++) { - candidate_values[candidate_pos] = values_buf[j]; - candidate_indices[candidate_pos] = indices_buf[j] + (int32_t) offset; - candidate_pos++; + for (uint32_t vi = 0; vi < bn; vi += 32) { + const uint32_t vn = MIN(32, bn - vi); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off + vi); + for (uint32_t i = 0; i < vn; i++) { + const float v = v_local.fp32[i]; + if (!(v > thr)) continue; + uint32_t pos = cnt < k ? cnt++ : k - 1; + while (pos > 0 && best_v[pos - 1] < v) { + best_v[pos] = best_v[pos - 1]; + best_i[pos] = best_i[pos - 1]; + pos--; + } + best_v[pos] = v; + best_i[pos] = (int32_t) (off + vi + i); + if (cnt == k) thr = best_v[k - 1]; + } } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) offset); } - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, 0); - if (merge_elems > candidate_pos) { - hvx_splat_f32_u((uint8_t *) (candidate_values + candidate_pos), -INFINITY, merge_elems - candidate_pos); - for (uint32_t j = candidate_pos; j < merge_elems; j++) { - candidate_indices[j] = 0; + // fewer than k values above -INF: take -INF or NaN elements, then pad with index 0 + for (uint32_t off = 0; off < n && cnt < k; off += 32) { + const uint32_t vn = MIN(32, n - off); + HVX_VectorAlias v_local; + v_local.v = hvx_vmem(vals + off); + for (uint32_t j = 0; j < vn && cnt < k; j++) { + if (!(v_local.fp32[j] > -INFINITY)) { + best_v[cnt] = -INFINITY; + best_i[cnt] = (int32_t) (off + j); + cnt++; + } } } - - bitonic_sort_vtcm_desc((uint8_t *) candidate_values, (uint8_t *) candidate_indices, merge_n_vec_pow2, false); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, 0); + for (; cnt < k; cnt++) { + best_v[cnt] = -INFINITY; + best_i[cnt] = 0; + } for (uint32_t j = 0; j < k; j++) { - out_values[j] = candidate_values[j]; - out_indices[j] = candidate_indices[j] + (int32_t) chunk_base; + out_v[j] = best_v[j]; + out_i[j] = best_i[j]; } } @@ -509,10 +515,14 @@ static void htp_sort_chunk_job(unsigned int n, unsigned int i, void * data) { } if (local_k > 1 && local_k <= 64 && chunk_elems > 1024) { - top_k_select_tiled(octx->ctx->dma[i], tr, src_addr, real_count, local_k, - values_buf, indices_buf, - &merge_values[i * local_k], &merge_indices[i * local_k], - chunk_base); + dma_sync_read(octx->ctx->dma[i], values_buf, src_addr, real_count * sizeof(float)); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + top_k_scan(values_buf, real_count, local_k, &merge_values[i * local_k], &merge_indices[i * local_k]); + for (uint32_t j = 0; j < local_k; j++) { + merge_indices[i * local_k + j] += (int32_t) chunk_base; + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); return; } @@ -668,10 +678,14 @@ static inline void compute_row_sort( uint32_t ne00, uint32_t chunk_elems, bool is_top_k, - enum ggml_sort_order order + enum ggml_sort_order order, + uint32_t k ) { if (is_top_k) { - if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { + if (k > 1 && k <= 64 && ne00 > 1024) { + float out_v[64]; + top_k_scan(cur_values, ne00, k, out_v, cur_indices); + } else if (ne00 == 32 || ne00 == 64 || ne00 == 128 || ne00 == 256 || ne00 == 512 || ne00 == 1024) { switch (ne00) { case 32: sort32_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; case 64: sort64_f32_hvx((uint8_t *) cur_values, (uint8_t *) cur_indices, GGML_SORT_ORDER_DESC); break; @@ -788,7 +802,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_queue_pop(dma_q); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order); + compute_row_sort(cur_values, cur_indices, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); // Push writeback of current slot @@ -817,7 +831,7 @@ static void htp_sort_multi_row_job(unsigned int n, unsigned int ith, void * data dma_sync_read(dma_q, values_buf, src_addr, src_bytes); htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); - compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order); + compute_row_sort(values_buf, indices_buf, ne00, chunk_elems, is_top_k, order, k); htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); dma_sync_write(dma_q, dst_addr, indices_buf, dst_bytes); From 6cb3e672e8d10fd4feab1a048bf2967c40623133 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Tue, 3 Nov 2026 10:30:37 -0800 Subject: [PATCH 033/286] gemma4: graph optimizations (massive gains on pp for E2B and E4B) (#31356) * Optimizing Gemma4 graph for prompt processing TODO: - Currently complete microbatch needs to contain tokens which does not require the logits; need to optimize this further - There are a few more operations that are performed after the kv cache is calculated, need to safely optimize them. * Not filtering the rows when they are unmasked * Cropping out the attention mask too. * gemma-4: improved pruning, mask cropping and PLE scaling Prune before the first KV-shared layer with the single-stream, causal, non-embedding safety gates. Use cropped masks only during shared-layer attention. Preserve requested layer-input embeddings in the KV prefix. Remove the PLE contiguous materialization and fold equivalent PLE scaling into RMSNorm parameters/weights. * spec-mtp: Enable masked h_nextn extraction only when the draft context shares the target KV cache This lets Gemma 4 prune unused prompt rows before its shared layers so that we keep the full perf P.P gains with MTP. Avoid extracting target hidden states for output-free prompt chunks in that shared-MTP path. When a batch has a selected final token, extract only that row unless the whole verification batch is selected. Keep ordinary MTP and classic speculative decoding behavior unchanged: they still use unmasked h_nextn because the draft context must catch up on all target rows. speculative-simple needs an update to select the final prompt row only for shared-KV MTP, so the optimization works there without changing other speculative modes. And it looks like the server already selects its final prompt row for first-token sampling. * spec-mtp: make sure to not drop first draft round * gemma4: enable pruning based on static cparams.kv_unified/n_seq_max=1 * gemma4-mtp: simplify prunning logic and avoid graph branches/re-allocs * gemma4: update pruning gates to use only n_seq_max==1, kv_unified can now be used for multi-seq * spec-simple: revert spec-simple changes for now * gemma4: remove causal_attn from pruning gate (last source of reallocs) * cont : adapt to llm_attn_mask changes * cont : add TODO --------- Co-authored-by: Meet Amin Co-authored-by: Georgi Gerganov --- common/speculative.cpp | 28 ++++++++++++----- src/models/gemma4.cpp | 69 +++++++++++++++++++++++++++++++----------- 2 files changed, 71 insertions(+), 26 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index f82dc879f1b8..e6290eaad776 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1613,12 +1613,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } } - llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ false); - llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); - is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; chain_heads = n_mtp_layers > 1 && !is_mem_shared; + llama_set_embeddings_nextn(ctx_tgt, true, /*masked*/ is_mem_shared); + llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); + if (chain_heads) { this->params.n_max = std::min(this->params.n_max, n_mtp_layers); @@ -1772,16 +1772,28 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; - verify_h_rows[seq_id] = n_rows; - verify_h[seq_id].resize((size_t) n_rows * n_embd); + if (is_mem_shared && !batch_in.tokens[i_batch_end[seq_id]].output) { + verify_h_rows[seq_id] = 0; + continue; + } + + const bool all_rows_output = std::all_of( + batch_in.tokens.begin() + i_batch_beg[seq_id], + batch_in.tokens.begin() + i_batch_end[seq_id] + 1, + [](const common_batch::token & token) { return token.output; }); + const int32_t h_rows = is_mem_shared && !all_rows_output ? 1 : n_rows; + const int32_t h_offset = is_mem_shared && !all_rows_output ? i_batch_end[seq_id] : i_batch_beg[seq_id]; + + verify_h_rows[seq_id] = h_rows; + verify_h[seq_id].resize((size_t) h_rows * n_embd); - for (int32_t i = 0; i < n_rows; ++i) { - const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, i_batch_beg[seq_id] + i); + for (int32_t i = 0; i < h_rows; ++i) { + const float * h = llama_get_embeddings_nextn_ith(ctx_tgt, h_offset + i); std::memcpy(verify_h[seq_id].data() + (size_t) i * n_embd, h, row_bytes); } std::memcpy(pending_h[seq_id].data(), - verify_h[seq_id].data() + (size_t) (n_rows - 1) * n_embd, row_bytes); + verify_h[seq_id].data() + (size_t) (h_rows - 1) * n_embd, row_bytes); } return true; diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cpp index d0b2b3a5b473..df245f81103e 100644 --- a/src/models/gemma4.cpp +++ b/src/models/gemma4.cpp @@ -154,8 +154,7 @@ std::unique_ptr llama_model_gemma4::build_arch_graph(const ll // get 2D slice view from a 3D tensor, the idx corresponds to the 3rd dim static ggml_tensor * gemma4_view_2d_slice(ggml_context * ctx0, ggml_tensor * x, int idx) { GGML_ASSERT(idx < (int) x->ne[2]); - return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], ggml_row_size(x->type, x->ne[0]), - idx * x->ne[0] * x->ne[1] * ggml_element_size(x)); + return ggml_view_2d(ctx0, x, x->ne[0], x->ne[1], x->nb[1], idx * x->nb[2]); } llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_params & params) : @@ -183,10 +182,14 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para // TODO: is causal == true correct? might need some changes auto * inp_attn = build_attn_inp_kv_iswa(); + auto inp_attn_full = *inp_attn; + auto inp_attn_prune = *inp_attn; + ggml_tensor * inp_out_ids = build_inp_out_ids(); if (model.per_layer_tok_embd) { - const float tok_embd_scale = sqrtf((float) n_embd_per_layer); + // fold per_layer_input_scale (1/sqrt(2)) from project_per_layer_inputs() into tok_embd_scale + const float tok_embd_scale = sqrtf((float) n_embd_per_layer) / sqrtf(2.0f); inp_per_layer = ggml_scale (ctx0, inp_per_layer, tok_embd_scale); inp_per_layer = ggml_reshape_3d(ctx0, inp_per_layer, n_embd_per_layer, n_layer, inp_per_layer->ne[1]); @@ -195,7 +198,36 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para inp_per_layer = project_per_layer_inputs(inpL, inp_per_layer); } + // KV-shared layers do not write to the KV cache, so prune non-output token rows before the first KV-shared layer + bool prune_shared = crop_before_nextn(inp_out_ids) && + hparams.n_layer_kv_from_start > 0 && hparams.n_layer_kv_from_start < n_layer && + !cparams.embeddings && cparams.n_seq_max == 1; + for (int il = (int) hparams.n_layer_kv_from_start; prune_shared && il <= n_layer; ++il) { + prune_shared = !cparams.embeddings_layer_inp[il]; + } + + int64_t n_rows = n_tokens; + for (int il = 0; il < n_layer; ++il) { + if (prune_shared && il == (int) hparams.n_layer_kv_from_start) { + inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); + inp_pos = ggml_reshape_1d(ctx0, ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, inp_pos, 1, n_tokens), inp_out_ids), n_outputs); + + // TODO: instead of doing get_rows here, extend llm_attn_mask to have a `out` mask that + // is construted during set_input to contain just the pruned output cells + // this can be used to enable pruning with more than 1 sequence, both with unified and split KV cache + // ref: https://github.com/ggml-org/llama.cpp/pull/31356#discussion_r4455286628 + inp_attn_prune.self_attn_mask.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask.raw, inp_out_ids); + inp_attn_prune.self_attn_mask_swa.raw = ggml_get_rows(ctx0, inp_attn_full.self_attn_mask_swa.raw, inp_out_ids); + + inp_attn_prune.self_attn_mask.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask.raw, GGML_TYPE_F16); + inp_attn_prune.self_attn_mask_swa.raw = ggml_cast(ctx0, inp_attn_prune.self_attn_mask_swa.raw, GGML_TYPE_F16); + + n_rows = n_outputs; + + inp_attn = &inp_attn_prune; + } + const int64_t n_embd_head = hparams.n_embd_head_k(il); GGML_ASSERT(n_embd_head == hparams.n_embd_head_v(il)); @@ -226,14 +258,14 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para qkv_fused = build_lora_mm(model.layers[il].wqkv, cur, model.layers[il].wqkv_s); cb(qkv_fused, "wqkv", il); const int64_t q_dim = n_embd_head * n_head; - Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, q_dim, n_tokens, qkv_fused->nb[1], 0)); + Qcur = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv_fused, q_dim, n_rows, qkv_fused->nb[1], 0)); } else { Qcur = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); } { cb(Qcur, "Qcur", il); - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens); + Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_rows); Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il); cb(Qcur, "Qcur_normed", il); @@ -287,9 +319,8 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para Qcur, nullptr, nullptr, nullptr, nullptr, nullptr, hparams.f_attention_scale, il); } - // TODO @ngxson : strip unused token right after the last KV layer to speed up prompt processing // keep all rows when extracting unmasked nextn embeddings (MTP target needs the hidden state for every token) - if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { + if (!prune_shared && il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { cur = ggml_get_rows(ctx0, cur, inp_out_ids); inpL = ggml_get_rows(ctx0, inpL, inp_out_ids); } @@ -331,7 +362,7 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * tmp = ggml_rms_norm(ctx0, attn_out, hparams.f_norm_rms_eps); tmp = ggml_scale(ctx0, tmp, 1.0f / sqrtf((float) n_embd)); tmp = ggml_mul(ctx0, tmp, model.layers[il].ffn_gate_inp_s); - ggml_tensor * logits = build_lora_mm(model.layers[il].ffn_gate_inp, tmp); // [n_expert, n_tokens] + ggml_tensor * logits = build_lora_mm(model.layers[il].ffn_gate_inp, tmp); // [n_expert, n_rows] cb(logits, "ffn_moe_logits", il); cur_moe = build_moe_ffn(cur_moe, @@ -383,18 +414,17 @@ llama_model_gemma4::graph::graph(const llama_model & model, const llm_graph_para ggml_tensor * pe_in = cur; cb(cur, "pe_in", il); - cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_tokens] + cur = build_lora_mm(model.layers[il].per_layer_inp_gate, cur); // [n_embd_per_layer, n_rows] cur = ggml_gelu(ctx0, cur); ggml_tensor * inp_this_layer = gemma4_view_2d_slice(ctx0, inp_per_layer, il); // [n_embd_per_layer, n_tokens] - // TODO @ngxson : improve this - if (il == n_layer - 1 && crop_before_nextn(inp_out_ids)) { + if ((prune_shared && !hparams.has_kv(il)) || (il == n_layer - 1 && crop_before_nextn(inp_out_ids))) { inp_this_layer = ggml_get_rows(ctx0, inp_this_layer, inp_out_ids); } cur = ggml_mul(ctx0, cur, inp_this_layer); - cur = build_lora_mm(model.layers[il].per_layer_proj, cur); // [n_embd, n_tokens] + cur = build_lora_mm(model.layers[il].per_layer_proj, cur); // [n_embd, n_rows] cur = build_norm(cur, model.layers[il].per_layer_post_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "per_layer_embd_out", il); @@ -522,19 +552,22 @@ ggml_tensor * llama_model_gemma4::graph::project_per_layer_inputs(ggml_tensor * const float per_layer_input_scale = 1.0f / sqrtf(2.0f); // note: this matrix multiplication will be performed in the input layer (i.e. on the CPU) + // fold per_layer_input_scale into proj_norm_w and per_layer_projection_scale into proj_eps + ggml_tensor * proj_norm_w = ggml_scale(ctx0, model.per_layer_proj_norm, per_layer_input_scale); + ggml_build_forward_expand(gf, proj_norm_w); + ggml_tensor * per_layer_proj; per_layer_proj = ggml_mul_mat (ctx0, model.per_layer_model_proj, inp_batch); - per_layer_proj = ggml_scale (ctx0, per_layer_proj, per_layer_projection_scale); per_layer_proj = ggml_reshape_3d(ctx0, per_layer_proj, n_embd_per_layer, n_layer, n_tokens); - per_layer_proj = build_norm(per_layer_proj, model.per_layer_proj_norm, nullptr, LLM_NORM_RMS, -1); + const float proj_eps = hparams.f_norm_rms_eps / (per_layer_projection_scale * per_layer_projection_scale); + per_layer_proj = ggml_rms_norm(ctx0, per_layer_proj, proj_eps); + per_layer_proj = ggml_mul(ctx0, per_layer_proj, proj_norm_w); cb(per_layer_proj, "per_layer_proj", -1); - inp_per_layer = ggml_add (ctx0, per_layer_proj, inp_per_layer); - inp_per_layer = ggml_scale(ctx0, inp_per_layer, per_layer_input_scale); + inp_per_layer = ggml_add(ctx0, per_layer_proj, inp_per_layer); cb(inp_per_layer, "inp_per_layer", -1); // permute to shape: [n_embd_per_layer, n_tokens, n_layer] - inp_per_layer = ggml_cont(ctx0, ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3)); - return inp_per_layer; + return ggml_permute(ctx0, inp_per_layer, 0, 2, 1, 3); } From 7071ba378afa96dcd69c41e1ec2b4912364dbee4 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Tue, 3 Nov 2026 12:04:03 -0800 Subject: [PATCH 034/286] hexagon: add TOP_K_MASK op (#31552) * hexagon: add TOP_K_MASK op * Simplify Hexagon TOP_K_MASK kernel and do not duplicate host configuration --- ggml/src/ggml-hexagon/htp-kparams.cpp | 192 ++++++ ggml/src/ggml-hexagon/htp-kparams.h | 1 + ggml/src/ggml-hexagon/htp-opnode.cpp | 9 + ggml/src/ggml-hexagon/htp/CMakeLists.txt | 1 + ggml/src/ggml-hexagon/htp/htp-ctx.h | 1 + ggml/src/ggml-hexagon/htp/htp-ops.h | 1 + ggml/src/ggml-hexagon/htp/main.c | 3 + ggml/src/ggml-hexagon/htp/top-k-mask-ops.c | 665 +++++++++++++++++++++ ggml/src/ggml-hexagon/htp/top-k-mask-ops.h | 131 ++++ tests/test-backend-ops.cpp | 7 + 10 files changed, 1011 insertions(+) create mode 100644 ggml/src/ggml-hexagon/htp/top-k-mask-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/top-k-mask-ops.h diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index f5c4ad7cb116..7362bb1c9f65 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -50,6 +50,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_OP_ROPE: return HTP_OP_ROPE; case GGML_OP_REPEAT: return HTP_OP_REPEAT; case GGML_OP_CUMSUM: return HTP_OP_CUMSUM; + case GGML_OP_TOP_K_MASK: return HTP_OP_TOP_K_MASK; case GGML_OP_FILL: return HTP_OP_FILL; case GGML_OP_DIAG: return HTP_OP_DIAG; case GGML_OP_SOLVE_TRI: return HTP_OP_SOLVE_TRI; @@ -3402,6 +3403,194 @@ static bool htp_kparams_solve_cumsum( return true; } +// Shrink until the layout fits: halve tile_cols down to 1024 columns, then drop threads. False below the VTCM floor. +static bool htp_top_k_mask_search(struct htp_top_k_mask_vtcm_layout * L, const struct htp_top_k_mask_shape * S, uint32_t kernel_type, + size_t vtcm_size, uint32_t * n_threads, uint32_t * tile_cols) { + const uint32_t min_cols = 1024; + while (true) { + htp_top_k_mask_vtcm_layout_build(L, S, kernel_type, *n_threads, *tile_cols); + if (L->total_bytes <= vtcm_size) { + return true; + } + if (*tile_cols > min_cols) { + *tile_cols = (std::max)(min_cols, (uint32_t) hex_round_up(*tile_cols / 2, 64)); + } else if (*n_threads > 1) { + (*n_threads)--; + } else { + return false; + } + } +} + +// Row r of t (same ne[1..3] as dst) is at r * nb[1]: the NPU needs one stride per input. +// A mask broadcasts by i mod ne: its rows are dense above nb[1], the divisors and dst ne[1..2] fit 16 bits. +static bool htp_top_k_mask_mask_ok(const struct ggml_tensor * m, const struct ggml_tensor * dst) { + return m->ne[1] <= UINT16_MAX && m->ne[2] <= UINT16_MAX && m->ne[3] <= UINT16_MAX && dst->ne[1] <= UINT16_MAX && dst->ne[2] <= UINT16_MAX && + (m->ne[2] == 1 || m->nb[2] == m->nb[1] * m->ne[1]) && + (m->ne[3] == 1 || m->nb[3] == m->nb[1] * m->ne[1] * m->ne[2]); +} + +static bool htp_top_k_mask_row_flat(const struct ggml_tensor * t, const struct ggml_tensor * dst) { + return t->ne[1] == dst->ne[1] && t->ne[2] == dst->ne[2] && t->ne[3] == dst->ne[3] && + (t->ne[2] == 1 || t->nb[2] == t->nb[1] * t->ne[1]) && + (t->ne[3] == 1 || t->nb[3] == t->nb[1] * t->ne[1] * t->ne[2]); +} + +static bool htp_kparams_solve_top_k_mask( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_top_k_mask_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * mask = op->src[0]; + const struct ggml_tensor * top_k = op->src[1]; + const struct ggml_tensor * score = op->src[2]; + const struct ggml_tensor * blocks = op->src[3]; + const struct ggml_tensor * extra = op->src[4]; + const struct ggml_tensor * dst = op; + + if ((mask->type != GGML_TYPE_F16 && mask->type != GGML_TYPE_F32) || dst->type != mask->type) { + return false; + } + const size_t elem_size = ggml_type_size(dst->type); + if (mask->nb[0] != elem_size || dst->nb[0] != elem_size || !ggml_is_contiguous(dst)) { + return false; + } + if (top_k->type != GGML_TYPE_I32 || top_k->nb[0] != 4) { + return false; + } + if (score && (score->type != GGML_TYPE_F32 || score->nb[0] != 4)) { + return false; + } + if (extra && (extra->type != GGML_TYPE_I32 || extra->nb[0] != 4)) { + return false; + } + // flat row addressing: inputs follow the dst rows, a mask with more than one row too + if (!htp_top_k_mask_row_flat(top_k, dst) || (score && !htp_top_k_mask_row_flat(score, dst)) || + (extra && !htp_top_k_mask_row_flat(extra, dst)) || (ggml_nrows(mask) > 1 && !htp_top_k_mask_mask_ok(mask, dst))) { + return false; + } + // the kernel scales block ids by blocks->nb[1] with a 16-bit vector multiply + if (blocks && (blocks->type != GGML_TYPE_I32 || blocks->nb[0] != 4 || blocks->nb[1] != (size_t) blocks->ne[0] * 4 || blocks->nb[1] > INT16_MAX)) { + return false; + } + + struct htp_top_k_mask_shape S = {}; + S.n_kv = (uint32_t) dst->ne[0]; + S.elem_size = (uint32_t) elem_size; + S.n_top = (uint32_t) top_k->ne[0]; + S.n_blk = blocks ? (uint32_t) blocks->ne[0] : 0; + S.n_pick = (uint32_t) (blocks ? blocks->ne[1] : score ? score->ne[0] : dst->ne[0]); + S.n_extra = extra ? (uint32_t) extra->ne[0] : 0; + const uint32_t total_rows = (uint32_t) ggml_nrows(dst); + S.has_score = score ? 1 : 0; + S.has_blocks = blocks ? 1 : 0; + S.mask_resident = ggml_nrows(mask) == 1 ? 1 : 0; + + // empty inputs would give out-of-range gathers and zero-length DMA rows: leave them to the CPU + if (S.n_kv == 0 || total_rows == 0 || S.n_pick == 0 || S.n_top == 0 || (blocks && S.n_blk == 0) || (extra && S.n_extra == 0)) { + return false; + } + + // PICKS for one row with a resident mask when its single row tile fits, ROWS otherwise + const uint32_t sess_threads = (std::max)(1u, htp_session_n_threads(sess)); + const size_t vtcm_size = htp_session_vtcm_size(sess); + + uint32_t kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + uint32_t n_threads = sess_threads; + uint32_t tile_cols = 0; + + struct htp_top_k_mask_vtcm_layout L; + bool found = false; + if (total_rows == 1 && S.mask_resident && sess_threads > 1) { + kernel_type = HTP_TOP_K_MASK_KERNEL_PICKS; + tile_cols = hex_round_up(S.n_kv, 64); + htp_top_k_mask_vtcm_layout_build(&L, &S, kernel_type, n_threads, tile_cols); + found = L.total_bytes <= vtcm_size; + } + if (!found) { + kernel_type = HTP_TOP_K_MASK_KERNEL_ROWS; + n_threads = (std::max)(1u, (std::min)(sess_threads, total_rows)); + tile_cols = hex_round_up(S.n_kv, 64); + found = htp_top_k_mask_search(&L, &S, kernel_type, vtcm_size, &n_threads, &tile_cols); + } + if (!found) { + return false; + } + + // compact kernel_params fields: sizes and offsets in HTP_TOP_K_MASK_UNIT bytes as u16, all multiples of the unit + const uint64_t off_extra = htp_top_k_mask_rowin_off_extra(S.n_top); + const uint64_t off_score = htp_top_k_mask_rowin_off_score(S.n_top, S.n_extra); + const uint64_t u16_max = (uint64_t) UINT16_MAX * HTP_TOP_K_MASK_UNIT; + if (L.rowin_bytes > u16_max || L.tile_bytes > u16_max || off_extra > u16_max || off_score > u16_max || L.off_scratch > u16_max) { + return false; + } + const auto units = [](uint64_t bytes) { return (uint16_t) (bytes / HTP_TOP_K_MASK_UNIT); }; + + kparams->n_threads = (uint8_t) n_threads; + kparams->kernel_type = (uint8_t) kernel_type; + kparams->flags = (dst->type == GGML_TYPE_F16 ? HTP_TOP_K_MASK_F16 : 0) | + (score ? HTP_TOP_K_MASK_HAS_SCORE : 0) | + (blocks ? HTP_TOP_K_MASK_HAS_BLOCKS : 0) | + (extra ? HTP_TOP_K_MASK_HAS_EXTRA : 0) | + (S.mask_resident ? HTP_TOP_K_MASK_MASK_RESIDENT : 0); + kparams->elem_size = (uint8_t) elem_size; + kparams->vtcm_size = (uint32_t) L.total_bytes; + kparams->n_kv = S.n_kv; + kparams->n_top = S.n_top; + kparams->n_pick = S.n_pick; + kparams->n_extra = S.n_extra; + kparams->ninf = dst->type == GGML_TYPE_F16 ? 0xfc00fc00u : 0xff800000u; + kparams->blocks_bytes = (uint32_t) L.blocks_bytes; + kparams->score_bytes = score ? S.n_pick * 4 : 0; + kparams->tile_cols = tile_cols; + kparams->n_col_tiles = (S.n_kv + tile_cols - 1) / tile_cols; + kparams->n_blk = (uint16_t) S.n_blk; + kparams->blocks_nb1 = (uint16_t) (blocks ? blocks->nb[1] : 0); + kparams->off_blocks_u = units(L.off_blocks); + kparams->off_mask_u = units(L.off_mask); + kparams->off_rowin_u = units(L.off_rowin); + kparams->off_tiles_u = units(L.off_tiles); + kparams->off_mtiles_u = units(L.off_mtiles); + kparams->off_scratch_u = units(L.off_scratch); + kparams->rowin_units = units(L.rowin_bytes); + kparams->tile_units = units(L.tile_bytes); + kparams->off_extra_units = units(off_extra); + kparams->off_score_units = units(off_score); + + if (kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS) { + // per thread slices of the static copies and of the flag clear, a multiple of the unit + const auto slice = [&](uint64_t n) { return units(hex_round_up((n + n_threads - 1) / n_threads, HTP_TOP_K_MASK_UNIT)); }; + kparams->u.picks.picks_per_thread = (uint32_t) hex_round_up((S.n_top + n_threads - 1) / n_threads, 32); + kparams->u.picks.blocks_slice_u = slice(L.blocks_bytes); + kparams->u.picks.top_slice_u = slice((uint64_t) S.n_top * 4); + kparams->u.picks.extra_slice_u = slice((uint64_t) S.n_extra * 4); + kparams->u.picks.score_slice_u = slice(kparams->score_bytes); + kparams->u.picks.flags_slice_u = slice(L.tile_bytes); + } else { + kparams->u.rows.total_rows = total_rows; + kparams->u.rows.rs_top = (uint32_t) top_k->nb[1]; + kparams->u.rows.rs_extra = extra ? (uint32_t) extra->nb[1] : 0; + kparams->u.rows.rs_score = score ? (uint32_t) score->nb[1] : 0; + kparams->u.rows.rs_dst = (uint32_t) dst->nb[1]; + kparams->u.rows.mask_nb1 = (uint32_t) mask->nb[1]; + if (S.mask_resident) { + kparams->u.rows.m.mask_bytes = (uint32_t) L.mask_bytes; + } else { + const int64_t d[5] = { dst->ne[1], dst->ne[2], mask->ne[1], mask->ne[2], mask->ne[3] }; + for (int k = 0; k < 5; k++) { + const struct fastdiv_values fd = init_fastdiv_values((uint32_t) d[k]); + kparams->u.rows.m.dec.mp[k] = fd.mp; + kparams->u.rows.m.dec.l[k] = (uint8_t) fd.l; + kparams->u.rows.m.dec.ne[k] = (uint16_t) d[k]; + } + } + } + + return true; +} + static bool htp_kparams_solve_fill( const struct ggml_hexagon_session * sess, const struct ggml_tensor * op, @@ -4807,6 +4996,9 @@ bool htp_kparams_solve( case HTP_OP_CUMSUM: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_TOP_K_MASK: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_FILL: return htp_kparams_invoke(sess, op, kparams); diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h index 9f2896eb507e..70616fbf5ae8 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.h +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -29,6 +29,7 @@ #include "htp/solve-tri-ops.h" #include "htp/ssm-conv.h" #include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" #include "htp/unary-ops.h" #include "htp/upscale-ops.h" diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp index 36e7a3f38c2f..6aee385badc4 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.cpp +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -19,6 +19,7 @@ #include "htp/solve-tri-ops.h" #include "htp/ssm-conv.h" #include "htp/sum-rows-ops.h" +#include "htp/top-k-mask-ops.h" #include "htp/unary-ops.h" #include "htp/upscale-ops.h" @@ -224,6 +225,14 @@ static std::string format_kernel_params(const htp_opnode & node) { } else if (node.opcode == HTP_OP_CUMSUM) { const auto * kparams = node.kparams(); snprintf(str, max_size, "%s nth %d vtcm %u", kparams->use_dma ? "dma" : "hvx", (int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_TOP_K_MASK) { + const auto * kparams = node.kparams(); + const char * ktype_str = "?"; + switch (kparams->kernel_type) { + case HTP_TOP_K_MASK_KERNEL_ROWS: ktype_str = "rows"; break; + case HTP_TOP_K_MASK_KERNEL_PICKS: ktype_str = "picks"; break; + } + snprintf(str, max_size, "%s nth %d tile %u x%u vtcm %u", ktype_str, (int) kparams->n_threads, (unsigned int) kparams->tile_cols, (unsigned int) kparams->n_col_tiles, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_FILL) { const auto * kparams = node.kparams(); snprintf(str, max_size, "%s nth %d", kparams->kernel_type == HTP_FILL_KERNEL_CONTIGUOUS ? "contig" : "strided", (int) kparams->n_threads); diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 7c7c5be17ce1..3dcf4445f603 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(${HTP_LIB} SHARED repeat-ops.c ssm-conv.c cumsum-ops.c + top-k-mask-ops.c fill-ops.c concat-ops.c diag-ops.c diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index 4e2386bfa1ab..e6e6254d2cdb 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -187,6 +187,7 @@ int op_argsort(struct htp_ops_context * octx); int op_top_k(struct htp_ops_context * octx); int op_ssm_conv(struct htp_ops_context * octx); int op_cumsum(struct htp_ops_context * octx); +int op_top_k_mask(struct htp_ops_context * octx); int op_fill(struct htp_ops_context * octx); int op_concat(struct htp_ops_context * octx); int op_diag(struct htp_ops_context * octx); diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 1e132f877952..99557672ee31 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -119,6 +119,7 @@ enum htp_op_code { HTP_OP_UPSCALE, HTP_OP_DSV4_HC_PRE, HTP_OP_DSV4_HC_POST, + HTP_OP_TOP_K_MASK, HTP_OP_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 4c82c684dc66..6a6d5a3a97ff 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -1008,6 +1008,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_CUMSUM: return op_cumsum(octx); + case HTP_OP_TOP_K_MASK: + return op_top_k_mask(octx); + case HTP_OP_FILL: return op_fill(octx); diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c new file mode 100644 index 000000000000..215f01ced76a --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.c @@ -0,0 +1,665 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include +#include + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "hex-common.h" +#include "hex-profile.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "htp-vtcm.h" +#include "hvx-types.h" +#include "hvx-utils.h" +#include "dma-queue.h" +#include "top-k-mask-ops.h" + +#define TOP_K_MASK_NINF_F32 0xff800000u // f32 score compare + +struct htp_top_k_mask_context { + struct htp_ops_context * octx; + const struct htp_top_k_mask_kernel_params * kparams; + + uint32_t rows_per_thread; + uint32_t row0, row1; // rows this device handles + uint32_t off_extra, off_score; // parts inside one row input slot + uint32_t col0, ncols_out; // PICKS: columns this device writes back + atomic_uint barrier; // PICKS: arrivals, barrier k waits for k * n_threads + + uint8_t * blocks; + uint8_t * mask; + uint8_t * rowin; + uint8_t * tiles; + uint8_t * mtiles; + uint8_t * scratch; +}; + +// DMA queue pop with a trace event around the wait +static inline void top_k_mask_pop(dma_queue * dma_q, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_DMA, info); + dma_queue_pop(dma_q); + htp_trace_event_stop(tr, HTP_TRACE_EVT_DMA, info); +} + +// Lanes 0..31 as words, in registers: inclusive scan of ones, minus one +static inline HVX_Vector top_k_mask_lane_iota(void) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector one = Q6_V_vsplat_R(1); + HVX_Vector v = one; + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 4)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 8)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 16)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 32)); + v = Q6_Vw_vadd_VwVw(v, Q6_V_vlalign_VVR(v, zero, 64)); + return Q6_Vw_vsub_VwVw(v, one); +} + +static __attribute__((noinline)) void top_k_mask_fill(uint8_t * restrict dst, uint32_t nbytes, uint32_t value) { + const HVX_Vector v = Q6_V_vsplat_R(value); + HVX_Vector * vdst = (HVX_Vector *) dst; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vdst[i] = v; + } +} + +#define TOP_K_MASK_DEAD 0x7fffffffu // cell value of a dead lane: c - c0 >=u ncols for every tile, signed max + +// Byte offsets of the cells in [c0, c0 + ncols) relative to c0, 0 for the others +static inline HVX_Vector top_k_mask_keep_off(HVX_Vector c, HVX_Vector v_c0, HVX_Vector v_ncols, const bool is_f16) { + const HVX_Vector cr = Q6_Vw_vsub_VwVw(c, v_c0); + return Q6_Vw_vasl_VwR(Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VuwVuw(v_ncols, cr), cr, Q6_V_vzero()), is_f16 ? 1 : 2); +} + +// Mask value of each cell lane in [c0, c0 + ncols) into tmp (predicated gather), returns the scatter offsets +static inline HVX_Vector top_k_mask_keep_gather(HVX_Vector c, const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector * restrict tmp, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + const HVX_Vector off = top_k_mask_keep_off(c, v_c0, v_ncols, is_f16); + + // Halfword data with word offsets: both halves of the pair hold the same offsets, the word predicate covers both halfwords of a lane + if (is_f16) { + Q6_vgather_AQRMWw(tmp, q, (size_t) mrow, mu, Q6_W_vcombine_VV(off, off)); + } else { + Q6_vgather_AQRMVw(tmp, q, (size_t) mrow, mu, off); + } + return off; +} + +static inline void top_k_mask_keep_scatter(HVX_Vector c, HVX_Vector off, uint8_t * restrict tile, HVX_Vector v_c0, HVX_Vector v_ncols, + uint32_t mu, HVX_Vector val, const bool is_f16) { + const HVX_VectorPred q = Q6_Q_vcmp_gt_VuwVuw(v_ncols, Q6_Vw_vsub_VwVw(c, v_c0)); + if (is_f16) { + Q6_vscatter_QRMWwV(q, (size_t) tile, mu, Q6_W_vcombine_VV(off, off), val); + } else { + Q6_vscatter_QRMVwV(q, (size_t) tile, mu, off, val); + } +} + +// Keep 4 cell vectors (dead lanes = TOP_K_MASK_DEAD): tile[c - c0] = mrow[c - c0] for c in [c0, c0 + ncols). One gather sync. +// flag: no mask, tile[c - c0] = all ones (keep flag) +static inline void top_k_mask_keep4(HVX_Vector c0v, HVX_Vector c1v, HVX_Vector c2v, HVX_Vector c3v, uint8_t * restrict tile, + const uint8_t * restrict mrow, HVX_Vector v_c0, HVX_Vector v_ncols, uint32_t mu, + HVX_Vector * restrict tmp, const bool is_f16, const bool flag) { + if (flag) { + const HVX_Vector ones = Q6_V_vsplat_R(0xffffffffu); + top_k_mask_keep_scatter(c0v, top_k_mask_keep_off(c0v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c1v, top_k_mask_keep_off(c1v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c2v, top_k_mask_keep_off(c2v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + top_k_mask_keep_scatter(c3v, top_k_mask_keep_off(c3v, v_c0, v_ncols, is_f16), tile, v_c0, v_ncols, mu, ones, is_f16); + return; + } + const HVX_Vector o0 = top_k_mask_keep_gather(c0v, mrow, v_c0, v_ncols, mu, tmp + 0, is_f16); + const HVX_Vector o1 = top_k_mask_keep_gather(c1v, mrow, v_c0, v_ncols, mu, tmp + 1, is_f16); + const HVX_Vector o2 = top_k_mask_keep_gather(c2v, mrow, v_c0, v_ncols, mu, tmp + 2, is_f16); + const HVX_Vector o3 = top_k_mask_keep_gather(c3v, mrow, v_c0, v_ncols, mu, tmp + 3, is_f16); + hvx_gather_sync(tmp + 3); + top_k_mask_keep_scatter(c0v, o0, tile, v_c0, v_ncols, mu, tmp[0], is_f16); + top_k_mask_keep_scatter(c1v, o1, tile, v_c0, v_ncols, mu, tmp[1], is_f16); + top_k_mask_keep_scatter(c2v, o2, tile, v_c0, v_ncols, mu, tmp[2], is_f16); + top_k_mask_keep_scatter(c3v, o3, tile, v_c0, v_ncols, mu, tmp[3], is_f16); +} + +// Pick vector j: picks below j1 and n_pick are valid, returns them with dead lanes marked. v_rem = j1 - lane index. +static inline HVX_Vector top_k_mask_load_picks(const int32_t * restrict tk, uint32_t j, HVX_Vector v_rem, HVX_Vector v_npick) { + const HVX_Vector p_raw = hvx_vmem(tk + j); + // lane + j = n_pick + const HVX_VectorPred valid = Q6_Q_vcmp_gtand_QVuwVuw(Q6_Q_vcmp_gt_VwVw(v_rem, Q6_V_vsplat_R(j)), v_npick, p_raw); + return Q6_V_vmux_QVV(valid, p_raw, Q6_V_vsplat_R(TOP_K_MASK_DEAD)); +} + +// Scatter the kept cells of picks [j0, j1) (and the extras) into the tile of columns [c0, c0 + ncols). slot holds the row inputs, mrow the mask columns. +// Gathers are issued in batches with one sync per batch: tmp[0..3] scores/blocks, tmp[4] score with blocks, tmp[5..8] mask values. +static inline __attribute__((always_inline)) void top_k_mask_scatter_tile( + const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, const uint8_t * restrict mrow, uint8_t * restrict tile, + HVX_Vector * restrict tmp, uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra, + const bool is_f16, const bool flag) { + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + + const int32_t * restrict tk = (const int32_t *) slot; + const int32_t * restrict ex = (const int32_t *) (slot + tctx->off_extra); + const uint8_t * restrict sc = slot + tctx->off_score; + const uint8_t * restrict blk = tctx->blocks; + + const bool has_score = kp->flags & HTP_TOP_K_MASK_HAS_SCORE; + const bool has_blocks = kp->flags & HTP_TOP_K_MASK_HAS_BLOCKS; + const uint32_t n_blk = kp->n_blk; + const uint32_t n_extra = do_extra ? kp->n_extra : 0; + const uint32_t mu = ncols * kp->elem_size - 1; + const uint32_t sc_mu = kp->score_bytes - 1; + const uint32_t blk_mu = kp->blocks_bytes - 1; + const uint32_t nb1_h = ((uint32_t) kp->blocks_nb1 << 16) | kp->blocks_nb1; + + const HVX_Vector v_lane = top_k_mask_lane_iota(); // in the HVX_COMP scope of the caller + const HVX_Vector v_c0 = Q6_V_vsplat_R(c0); + const HVX_Vector v_ncols = Q6_V_vsplat_R(ncols); + const HVX_Vector v_npick = Q6_V_vsplat_R(kp->n_pick); + const HVX_Vector v_pmax = Q6_V_vsplat_R(kp->n_pick - 1); // dead picks gather the last entry, every gather stays in its region + const HVX_Vector v_rem = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(j1), v_lane); + const HVX_Vector v_ninf = Q6_V_vsplat_R(TOP_K_MASK_NINF_F32); + const HVX_Vector v_dead = Q6_V_vsplat_R(TOP_K_MASK_DEAD); + + HVX_Vector * restrict mtmp = tmp + 5; + + if (has_blocks) { + for (uint32_t j = j0; j < j1; j += VLEN_FP32) { + HVX_Vector cell = top_k_mask_load_picks(tk, j, v_rem, v_npick); + const HVX_Vector p = Q6_Vw_vmin_VwVw(cell, v_pmax); + if (has_score) { + Q6_vgather_ARMVw(tmp + 4, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(p, 2)); + } + const HVX_Vector boff = Q6_Vw_vmpyi_VwRh(p, nb1_h); + for (uint32_t k = 0; k < n_blk; k += 4) { + // blocks k..k+3 of every pick; the missing ones (k + i >= n_blk) are dead + const uint32_t nk = MIN(n_blk - k, 4); + const HVX_Vector bo = Q6_Vw_vadd_VwVw(boff, Q6_V_vsplat_R(k * 4)); + for (uint32_t i = 0; i < nk; i++) { + Q6_vgather_ARMVw(tmp + i, (size_t) blk, blk_mu, Q6_Vw_vadd_VwVw(bo, Q6_V_vsplat_R(i * 4))); + } + hvx_gather_sync(tmp + nk - 1); + if (has_score && k == 0) { + // IEEE compare: NaN and -inf scores are dead + cell = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[4], v_ninf), cell, v_dead); + } + const HVX_VectorPred dead = Q6_Q_vcmp_eq_VwVw(cell, v_dead); + top_k_mask_keep4(Q6_V_vmux_QVV(dead, v_dead, tmp[0]), + nk > 1 ? Q6_V_vmux_QVV(dead, v_dead, tmp[1]) : v_dead, + nk > 2 ? Q6_V_vmux_QVV(dead, v_dead, tmp[2]) : v_dead, + nk > 3 ? Q6_V_vmux_QVV(dead, v_dead, tmp[3]) : v_dead, + tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + } else { + // 4 pick vectors per batch, the vectors at or past j1 are dead and not loaded + for (uint32_t j = j0; j < j1; j += 4 * VLEN_FP32) { + HVX_Vector c0v = top_k_mask_load_picks(tk, j, v_rem, v_npick); + HVX_Vector c1v = j + 1 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 1 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c2v = j + 2 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 2 * VLEN_FP32, v_rem, v_npick) : v_dead; + HVX_Vector c3v = j + 3 * VLEN_FP32 < j1 ? top_k_mask_load_picks(tk, j + 3 * VLEN_FP32, v_rem, v_npick) : v_dead; + if (has_score) { + Q6_vgather_ARMVw(tmp + 0, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c0v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 1, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c1v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 2, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c2v, v_pmax), 2)); + Q6_vgather_ARMVw(tmp + 3, (size_t) sc, sc_mu, Q6_Vw_vasl_VwR(Q6_Vw_vmin_VwVw(c3v, v_pmax), 2)); + hvx_gather_sync(tmp + 3); + c0v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[0], v_ninf), c0v, v_dead); + c1v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[1], v_ninf), c1v, v_dead); + c2v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[2], v_ninf), c2v, v_dead); + c3v = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VsfVsf(tmp[3], v_ninf), c3v, v_dead); + } + top_k_mask_keep4(c0v, c1v, c2v, c3v, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + } + + const HVX_Vector v_rem_ex = Q6_Vw_vsub_VwVw(Q6_V_vsplat_R(n_extra), v_lane); + for (uint32_t j = 0; j < n_extra; j += VLEN_FP32) { + const HVX_VectorPred tail = Q6_Q_vcmp_gt_VwVw(v_rem_ex, Q6_V_vsplat_R(j)); + top_k_mask_keep4(Q6_V_vmux_QVV(tail, hvx_vmem(ex + j), v_dead), v_dead, v_dead, v_dead, tile, mrow, v_c0, v_ncols, mu, mtmp, is_f16, flag); + } + + // release the scatters before DMA reads the tile + hvx_gather_sync(tile); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, false, false); +} + +static __attribute__((noinline)) void top_k_mask_scatter_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + const uint8_t * restrict mrow, uint8_t * restrict tile, HVX_Vector * restrict tmp, + uint32_t c0, uint32_t ncols, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, mrow, tile, tmp, c0, ncols, j0, j1, do_extra, true, false); +} + +// PICKS: keep flags (all ones) of the picks [j0, j1) into the flag tile of the whole row, the element size picks the flag width +static __attribute__((noinline)) void top_k_mask_flag_tile_f32(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, false, true); +} + +static __attribute__((noinline)) void top_k_mask_flag_tile_f16(const struct htp_top_k_mask_context * tctx, const uint8_t * restrict slot, + uint8_t * restrict flags, HVX_Vector * restrict tmp, uint32_t j0, uint32_t j1, bool do_extra) { + top_k_mask_scatter_tile(tctx, slot, NULL, flags, tmp, 0, tctx->kparams->n_kv, j0, j1, do_extra, true, true); +} + +// out = flag ? out : -inf over nbytes (a multiple of 128) of the row tile +static __attribute__((noinline)) void top_k_mask_select(uint8_t * restrict out, const uint8_t * restrict flags, uint32_t nbytes, uint32_t ninf) { + const HVX_Vector v_ninf = Q6_V_vsplat_R(ninf); + HVX_Vector * vout = (HVX_Vector *) out; + const HVX_Vector * vflag = (const HVX_Vector *) flags; + const uint32_t nvec = nbytes / VLEN; + + #pragma unroll(4) + for (uint32_t i = 0; i < nvec; i++) { + vout[i] = Q6_V_vxor_VV(v_ninf, Q6_V_vand_VV(Q6_V_vxor_VV(vout[i], v_ninf), vflag[i])); + } +} + +// Push the row inputs of flat row r into slot: top_k, extra, score. Returns the push count. +static inline uint32_t top_k_mask_push_rowin(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint8_t * slot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + const uint32_t tk_bytes = kp->n_top * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot, top_k->data + r * kp->u.rows.rs_top), tk_bytes, tk_bytes, tk_bytes, 1); + uint32_t n = 1; + + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + const uint32_t ex_bytes = kp->n_extra * sizeof(int32_t); + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_extra, extra->data + r * kp->u.rows.rs_extra), ex_bytes, ex_bytes, ex_bytes, 1); + n++; + } + + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + dma_queue_push(dma_q, dma_make_data(slot + tctx->off_score, score->data + r * kp->u.rows.rs_score), kp->score_bytes, kp->score_bytes, kp->score_bytes, 1); + n++; + } + + return n; +} + +static inline struct fastdiv_values top_k_mask_fd(const struct htp_top_k_mask_kernel_params * kp, uint32_t k) { + const struct fastdiv_values fd = { kp->u.rows.m.dec.mp[k], kp->u.rows.m.dec.l[k] }; + return fd; +} + +// Mask row of flat row r: (i1, i2, i3) of dst, each taken modulo the mask dims, rows dense above the mask ne1 +static inline uint32_t top_k_mask_mask_row(const struct htp_top_k_mask_kernel_params * kp, uint32_t r) { + const uint16_t * ne = kp->u.rows.m.dec.ne; + const struct fastdiv_values fd1 = top_k_mask_fd(kp, 0); + const struct fastdiv_values fd2 = top_k_mask_fd(kp, 1); + const struct fastdiv_values fm1 = top_k_mask_fd(kp, 2); + const struct fastdiv_values fm2 = top_k_mask_fd(kp, 3); + const struct fastdiv_values fm3 = top_k_mask_fd(kp, 4); + + const uint32_t q1 = fastdiv(r, &fd1); + const uint32_t i1 = r - q1 * ne[0]; + const uint32_t i3 = fastdiv(q1, &fd2); + const uint32_t i2 = q1 - i3 * ne[1]; + + const uint32_t m1 = fastmodulo(i1, ne[2], &fm1); + const uint32_t m2 = fastmodulo(i2, ne[3], &fm2); + const uint32_t m3 = fastmodulo(i3, ne[4], &fm3); + return (m3 * ne[3] + m2) * ne[2] + m1; +} + +// Push the inputs of item (row r, col tile ct): row inputs on the first tile of a row, then the mask tile. Returns the push count. +static inline uint32_t top_k_mask_push_item(const struct htp_top_k_mask_context * tctx, dma_queue * dma_q, uint32_t r, uint32_t ct, + uint8_t * slot, uint8_t * mslot) { + const struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * mask = octx->src[0]; + + const uint32_t c0 = ct * kp->tile_cols; + + uint32_t n = 0; + if (ct == 0) { + n += top_k_mask_push_rowin(tctx, dma_q, r, slot); + } + + if (mslot) { + const uint32_t m_off = top_k_mask_mask_row(kp, r) * kp->u.rows.mask_nb1 + c0 * kp->elem_size; + const uint32_t nb = MIN(kp->tile_cols, kp->n_kv - c0) * kp->elem_size; + dma_queue_push(dma_q, dma_make_data(mslot, mask->data + m_off), nb, nb, nb, 1); + n++; + } + + return n; +} + +static inline void top_k_mask_next(uint32_t * r, uint32_t * ct, uint32_t n_col_tiles) { + if (++(*ct) == n_col_tiles) { + *ct = 0; + ++(*r); + } +} + +// ROWS: the thread count follows the device rows (mdev), so the slice of an n byte copy is sized here +static inline uint32_t top_k_mask_slice(const struct htp_ops_context * octx, unsigned int nth, uint32_t n) { + return hex_round_up(fastdiv(n + nth - 1, &octx->n_threads_div), HTP_TOP_K_MASK_UNIT); +} + +// Push this thread's slice (128B aligned) of a contiguous n byte copy. Returns the push count. +static inline uint32_t top_k_mask_push_slice(dma_queue * dma_q, unsigned int ith, dma_addr_t dst, dma_addr_t src, uint32_t n, uint32_t slice) { + const uint32_t b0 = ith * slice; + if (b0 >= n) { + return 0; + } + const uint32_t nb = MIN(slice, n - b0); + dma_queue_push(dma_q, dma_make_data(dst + b0, src + b0), nb, nb, nb, 1); + return 1; +} + +// Pass 1 (ROWS): load the blocks table and the resident mask row, split across threads. Fill the tiles of this thread with -inf meanwhile. +static void top_k_mask_preload(unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + uint32_t n_pushed = 0; + + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, top_k_mask_slice(octx, nth, kp->blocks_bytes)); + } + if (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) { + const uint32_t mask_bytes = kp->u.rows.m.mask_bytes; + n_pushed += top_k_mask_push_slice(dma_q, ith, (dma_addr_t) tctx->mask, octx->src[0]->data, mask_bytes, top_k_mask_slice(octx, nth, mask_bytes)); + } + + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_fill(tctx->tiles + ith * 2 * tile_bytes, 2 * tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): depth-2 DMA pipeline over the items of rows [row_first, row_last), item = (row, col tile). +// The row inputs alternate between 2 slots by row. +static void top_k_mask_run_items(const struct htp_top_k_mask_context * tctx, unsigned int ith, uint32_t row_first, uint32_t row_last) { + struct htp_ops_context * octx = tctx->octx; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + const struct htp_tensor * dst = octx->dst; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const bool is_f16 = kp->flags & HTP_TOP_K_MASK_F16; + const uint32_t elem_size = kp->elem_size; + + const uint32_t rowin_bytes = kp->rowin_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + const uint32_t tile_cols = kp->tile_cols; + const uint32_t n_col_tiles = kp->n_col_tiles; + const uint32_t n_items = (row_last - row_first) * n_col_tiles; + + // 2 row input slots per thread, alternating by row + uint8_t * rowin = tctx->rowin + ith * 2 * rowin_bytes; + uint8_t * tiles = tctx->tiles + ith * 2 * tile_bytes; + uint8_t * mtiles = (kp->flags & HTP_TOP_K_MASK_MASK_RESIDENT) ? NULL : tctx->mtiles + ith * 2 * tile_bytes; + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + const dma_addr_t dst_data = dst->data; + + uint32_t n_in[2] = { 0, 0 }; + + // prefetch cursor + uint32_t pr = row_first, pct = 0; + + for (uint32_t it = 0; it < MIN(n_items, 2); it++) { + // Dummy dst writeback to establish queue ordering + dma_queue_push(dma_q, dma_make_data(dst_data, tiles + it * tile_bytes), 0, 0, 0, 0); + n_in[it] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + it * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + + uint32_t r = row_first, ct = 0; + + for (uint32_t it = 0; it < n_items; it++) { + const uint32_t s = it & 1; + uint8_t * tile = tiles + s * tile_bytes; + + // writeback of item it - 2 (or the dummy), then the tile is free again + top_k_mask_pop(dma_q, tr, (uint16_t) it); + if (it >= 2) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + top_k_mask_fill(tile, tile_bytes, kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + } + + for (uint32_t i = 0; i < n_in[s]; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) it); + } + + const uint32_t c0 = ct * tile_cols; + const uint32_t ncols = MIN(tile_cols, kp->n_kv - c0); + + const uint8_t * slot = rowin + ((r - row_first) & 1) * rowin_bytes; + const uint8_t * mrow = mtiles ? mtiles + s * tile_bytes : tctx->mask + c0 * elem_size; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + if (is_f16) { + top_k_mask_scatter_tile_f16(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } else { + top_k_mask_scatter_tile_f32(tctx, slot, mrow, tile, tmp, c0, ncols, 0, kp->n_top, true); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) it); + + const uint32_t d_off = r * kp->u.rows.rs_dst + c0 * elem_size; + const uint32_t nb = ncols * elem_size; + dma_queue_push(dma_q, dma_make_data(dst_data + d_off, tile), nb, nb, nb, 1); + + if (it + 2 < n_items) { + n_in[s] = top_k_mask_push_item(tctx, dma_q, pr, pct, rowin + ((pr - row_first) & 1) * rowin_bytes, mtiles ? mtiles + s * tile_bytes : NULL); + top_k_mask_next(&pr, &pct, n_col_tiles); + } + top_k_mask_next(&r, &ct, n_col_tiles); + } + + const uint32_t n_wb = MIN(n_items, 2); + for (uint32_t i = 0; i < n_wb; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } +} + +// Pass 2 (ROWS): whole rows split across threads, the col tiles of a row in order +static void top_k_mask_thread_rows(unsigned int nth, unsigned int ith, void * data) { + const struct htp_top_k_mask_context * tctx = (const struct htp_top_k_mask_context *) data; + + const uint32_t ir0 = tctx->row0 + ith * tctx->rows_per_thread; + const uint32_t ir1 = MIN(ir0 + tctx->rows_per_thread, tctx->row1); + if (ir0 >= ir1) { + return; + } + + top_k_mask_run_items(tctx, ith, ir0, ir1); +} + +static inline void top_k_mask_barrier(atomic_uint * b, uint32_t target, struct htp_thread_trace * tr, uint16_t info) { + htp_trace_event_start(tr, HTP_TRACE_EVT_FENCE, info); + atomic_fetch_add_explicit(b, 1, memory_order_release); + while (atomic_load_explicit(b, memory_order_acquire) < target) { + hex_pause(); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_FENCE, info); +} + +// PICKS, one pass over n_virt = kp->n_threads virtual workers (host slices); real worker ith runs the virtual workers ith, ith + nth, ... +// A: shared preload and a slice of the flag clear per virtual worker, the mask slice of this worker into the row tile; +// barrier; B: scatter keep flags of the picks of each virtual worker; barrier; C: select mask or -inf in this worker's slice and write it back +static void top_k_mask_thread_picks(unsigned int nth, unsigned int ith, void * data) { + struct htp_top_k_mask_context * tctx = (struct htp_top_k_mask_context *) data; + const struct htp_top_k_mask_kernel_params * kp = tctx->kparams; + struct htp_ops_context * octx = tctx->octx; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const struct htp_tensor * top_k = octx->src[1]; + const struct htp_tensor * score = octx->src[2]; + const struct htp_tensor * extra = octx->src[4]; + + uint8_t * slot = tctx->rowin; + const uint32_t tile_bytes = kp->tile_units * HTP_TOP_K_MASK_UNIT; + uint8_t * out = tctx->tiles; + uint8_t * flags = tctx->tiles + tile_bytes; + + const uint32_t n_virt = kp->n_threads; + + // this worker's slice of the device columns: the mask goes straight into the row tile, it is popped after the scan. + // The device share of the columns depends on mdev, so this slice is sized here. + const uint32_t out_b0 = tctx->col0 * kp->elem_size; + const uint32_t out_nb = tctx->ncols_out * kp->elem_size; + const uint32_t wslice = hex_round_up(fastdiv(out_nb + nth - 1, &octx->n_threads_div), 128); + const uint32_t wb0 = out_b0 + ith * wslice; + const uint32_t wnb = ith * wslice < out_nb ? MIN(wslice, out_nb - ith * wslice) : 0; + + const uint32_t fslice = kp->u.picks.flags_slice_u * HTP_TOP_K_MASK_UNIT; + + for (uint32_t v = ith; v < n_virt; v += nth) { + uint32_t n_pushed = 0; + if (kp->blocks_bytes) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) tctx->blocks, octx->src[3]->data, kp->blocks_bytes, kp->u.picks.blocks_slice_u * HTP_TOP_K_MASK_UNIT); + } + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) slot, top_k->data, kp->n_top * sizeof(int32_t), kp->u.picks.top_slice_u * HTP_TOP_K_MASK_UNIT); + if (kp->flags & HTP_TOP_K_MASK_HAS_EXTRA) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_extra), extra->data, kp->n_extra * sizeof(int32_t), kp->u.picks.extra_slice_u * HTP_TOP_K_MASK_UNIT); + } + if (kp->flags & HTP_TOP_K_MASK_HAS_SCORE) { + n_pushed += top_k_mask_push_slice(dma_q, v, (dma_addr_t) (slot + tctx->off_score), score->data, kp->score_bytes, kp->u.picks.score_slice_u * HTP_TOP_K_MASK_UNIT); + } + + // the mask slice follows the last preload in the queue, so the pops below leave it in flight + if (wnb && v + nth >= n_virt) { + dma_queue_push(dma_q, dma_make_data(out + wb0, octx->src[0]->data + wb0), wnb, wnb, wnb, 1); + } + + const uint32_t fb0 = v * fslice; + if (fb0 < tile_bytes) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + top_k_mask_fill(flags + fb0, MIN(fslice, tile_bytes - fb0), 0); // clear the flags + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + for (uint32_t i = 0; i < n_pushed; i++) { + top_k_mask_pop(dma_q, tr, (uint16_t) i); + } + } + + top_k_mask_barrier(&tctx->barrier, nth, tr, 0); + + HVX_Vector * tmp = (HVX_Vector *) (tctx->scratch + ith * HTP_TOP_K_MASK_SCRATCH_BYTES); + + for (uint32_t v = ith; v < n_virt; v += nth) { + const uint32_t j0 = MIN(v * kp->u.picks.picks_per_thread, kp->n_top); + const uint32_t j1 = MIN(j0 + kp->u.picks.picks_per_thread, kp->n_top); + const bool do_extra = v == n_virt - 1; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + if (kp->flags & HTP_TOP_K_MASK_F16) { + top_k_mask_flag_tile_f16(tctx, slot, flags, tmp, j0, j1, do_extra); + } else { + top_k_mask_flag_tile_f32(tctx, slot, flags, tmp, j0, j1, do_extra); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) v); + } + + top_k_mask_barrier(&tctx->barrier, 2 * nth, tr, 1); + + if (wnb) { + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + // wb0 is 128B aligned (col0 is a multiple of 64 columns), the select may run past the row inside the tile + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + top_k_mask_select(out + wb0, flags + wb0, hex_round_up(wnb, 128), kp->ninf); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) ith); + dma_queue_push(dma_q, dma_make_data(octx->dst->data + wb0, out + wb0), wnb, wnb, wnb, 1); + top_k_mask_pop(dma_q, tr, (uint16_t) ith); + } +} + +int op_top_k_mask(struct htp_ops_context * octx) { + const struct htp_top_k_mask_kernel_params * kp = (const struct htp_top_k_mask_kernel_params *) octx->kernel_params; + + if (kp->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + const bool picks = kp->kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + + // mdev: ROWS gives this device a contiguous share of the rows, DMA writes are exempt from the 128B rule. + // PICKS: every device scans the row, the write back is split by 64 column chunks. + uint32_t dev_row0 = 0; + uint32_t dev_rows = picks ? 1 : kp->u.rows.total_rows; + uint32_t dev_c0 = 0; + uint32_t dev_cols = kp->n_kv; + if (octx->ctx->mdev.count > 1) { + const struct htp_tensor_mdev_range range = picks ? + htp_tensor_mdev_partition(kp->n_kv, 64, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div) : + htp_tensor_mdev_partition(kp->u.rows.total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + if (range.count == 0) { + return HTP_STATUS_OK; + } + if (picks) { + dev_c0 = range.start; + dev_cols = range.count; + } else { + dev_row0 = range.start; + dev_rows = range.count; + } + } + + // fewer workers than kparams->n_threads always fit the VTCM layout: ROWS runs one row range per worker, PICKS loops over the host slices + const uint32_t max_threads = MIN((uint32_t) kp->n_threads, octx->ctx->n_threads); + if (!htp_ops_context_set_n_threads(octx, picks ? max_threads : MIN(max_threads, dev_rows))) { + return HTP_STATUS_INVAL_PARAMS; + } + const uint32_t n_threads = octx->n_threads; + + uint8_t * vtcm = octx->ctx->vtcm_base; + + struct htp_top_k_mask_context tctx = { + .octx = octx, + .kparams = kp, + .off_extra = kp->off_extra_units * HTP_TOP_K_MASK_UNIT, + .off_score = kp->off_score_units * HTP_TOP_K_MASK_UNIT, + .row0 = dev_row0, + .row1 = dev_row0 + dev_rows, + .col0 = dev_c0, + .ncols_out = dev_cols, + .blocks = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_blocks_u * HTP_TOP_K_MASK_UNIT), + .mask = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mask_u * HTP_TOP_K_MASK_UNIT), + .rowin = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_rowin_u * HTP_TOP_K_MASK_UNIT), + .tiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_tiles_u * HTP_TOP_K_MASK_UNIT), + .mtiles = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_mtiles_u * HTP_TOP_K_MASK_UNIT), + .scratch = VTCM_LAYOUT_PTR(uint8_t, vtcm, kp->off_scratch_u * HTP_TOP_K_MASK_UNIT), + }; + + if (picks) { + atomic_init(&tctx.barrier, 0); + work_queue_run(octx->ctx->work_queue, top_k_mask_thread_picks, &tctx, n_threads); + return HTP_STATUS_OK; + } + + tctx.rows_per_thread = fastdiv(dev_rows + n_threads - 1, &octx->n_threads_div); + + work_queue_run(octx->ctx->work_queue, top_k_mask_preload, &tctx, n_threads); + work_queue_run(octx->ctx->work_queue, top_k_mask_thread_rows, &tctx, n_threads); + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h new file mode 100644 index 000000000000..ba1be96b8091 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/top-k-mask-ops.h @@ -0,0 +1,131 @@ +#ifndef HTP_TOP_K_MASK_OPS_H +#define HTP_TOP_K_MASK_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" +#include +#include +#include + +// HVX gather results land here (gathers write to VTCM only): 10 vectors per thread +#define HTP_TOP_K_MASK_SCRATCH_BYTES 1280 + +enum htp_top_k_mask_kernel { + HTP_TOP_K_MASK_KERNEL_ROWS = 0, // threads split rows, row inputs streamed per row + HTP_TOP_K_MASK_KERNEL_PICKS = 1, // one row with one resident mask row: threads split the picks, keep flags select mask or -inf +}; + +#define HTP_TOP_K_MASK_F16 (1u << 0) +#define HTP_TOP_K_MASK_HAS_SCORE (1u << 1) +#define HTP_TOP_K_MASK_HAS_BLOCKS (1u << 2) +#define HTP_TOP_K_MASK_HAS_EXTRA (1u << 3) +#define HTP_TOP_K_MASK_MASK_RESIDENT (1u << 4) // mask has exactly 1 row + +#define HTP_TOP_K_MASK_UNIT 128 // unit of the compact u16 sizes in kernel_params + +struct htp_top_k_mask_shape { + uint32_t n_kv, elem_size, n_top, n_pick, n_blk, n_extra; + uint32_t has_score, has_blocks, mask_resident; +}; + +struct htp_top_k_mask_vtcm_layout { + size_t total_bytes; + size_t off_blocks, off_mask, off_rowin, off_tiles, off_mtiles; // 0 when unused + size_t off_scratch; // HTP_TOP_K_MASK_SCRATCH_BYTES per thread + size_t rowin_bytes; // one slot of top_k + extra + score row, parts at htp_top_k_mask_rowin_off_* + size_t tile_bytes; // one output tile slot, 128B aligned + size_t blocks_bytes, mask_bytes; // exact DMA sizes, 0 when unused +}; + +static inline uint64_t htp_top_k_mask_align128(uint64_t n) { + return (n + 127) & ~(uint64_t) 127; +} + +// Parts of one row input slot: top_k at 0, then extra, then score; used by the host layout +static inline uint64_t htp_top_k_mask_rowin_off_extra(uint32_t n_top) { + return htp_top_k_mask_align128((uint64_t) n_top * 4); +} + +static inline uint64_t htp_top_k_mask_rowin_off_score(uint32_t n_top, uint32_t n_extra) { + return htp_top_k_mask_rowin_off_extra(n_top) + htp_top_k_mask_align128((uint64_t) n_extra * 4); +} + +// the caller compares total_bytes against the VTCM size +static inline void htp_top_k_mask_vtcm_layout_build(struct htp_top_k_mask_vtcm_layout * L, + const struct htp_top_k_mask_shape * S, uint32_t kernel_type, uint32_t n_threads, uint32_t tile_cols) { + memset(L, 0, sizeof(*L)); + + // PICKS streams the mask straight into its row tile + const bool picks = kernel_type == HTP_TOP_K_MASK_KERNEL_PICKS; + const uint64_t blocks_exact = S->has_blocks ? (uint64_t) S->n_blk * S->n_pick * 4 : 0; + const uint64_t blocks_bytes = htp_top_k_mask_align128(blocks_exact); + const uint64_t mask_exact = S->mask_resident && !picks ? (uint64_t) S->n_kv * S->elem_size : 0; + const uint64_t mask_bytes = htp_top_k_mask_align128(mask_exact); + const uint64_t rowin_bytes = htp_top_k_mask_rowin_off_score(S->n_top, S->n_extra) + + (S->has_score ? htp_top_k_mask_align128((uint64_t) S->n_pick * 4) : 0); + const uint64_t tile_bytes = htp_top_k_mask_align128((uint64_t) tile_cols * S->elem_size); + + // ROWS streams row inputs into 2 slots per thread. PICKS: one row, a row tile and a flag tile. + const uint64_t rowin_total = (picks ? 1 : 2 * (uint64_t) n_threads) * rowin_bytes; + const uint64_t tiles_total = picks ? 2 * tile_bytes : 2 * (uint64_t) n_threads * tile_bytes; + const uint64_t mtiles_total = S->mask_resident ? 0 : tiles_total; + const uint64_t scratch = (uint64_t) n_threads * HTP_TOP_K_MASK_SCRATCH_BYTES; + + size_t off = 0; + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_blocks, (size_t) blocks_bytes, blocks_bytes > 0); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mask, (size_t) mask_bytes, mask_bytes > 0); + VTCM_LAYOUT_ALLOC(off, off_rowin, (size_t) rowin_total); + VTCM_LAYOUT_ALLOC(off, off_tiles, (size_t) tiles_total); + VTCM_LAYOUT_ALLOC_OPTIONAL(off, off_mtiles, (size_t) mtiles_total, !S->mask_resident); + VTCM_LAYOUT_ALLOC(off, off_scratch, (size_t) scratch); + + L->rowin_bytes = (size_t) rowin_bytes; + L->tile_bytes = (size_t) tile_bytes; + L->blocks_bytes = (size_t) blocks_exact; + L->mask_bytes = (size_t) mask_exact; + L->total_bytes = off; +} + +// Set by the host, used as is by the NPU: no tensor shape, stride or layout is derived on the NPU. +// Sizes of one slot or tile, the slot parts and the VTCM offsets are in HTP_TOP_K_MASK_UNIT bytes as u16. +// Row strides: byte offset of flat row r of top_k, extra, score and dst is r * stride (the host requires row flattenable inputs). +// PICKS splits the work in n_threads virtual workers; fewer real workers loop over them. +struct htp_top_k_mask_kernel_params { + uint8_t n_threads, kernel_type, flags, elem_size; + uint32_t vtcm_size; + uint32_t n_kv, n_top, n_pick, n_extra; + uint32_t ninf; // -inf cell pattern of elem_size + uint32_t blocks_bytes, score_bytes; // exact DMA sizes, gather regions are [0, bytes) + uint32_t tile_cols, n_col_tiles; + uint16_t n_blk, blocks_nb1; // blocks_nb1 = n_blk * 4, fits a signed 16-bit multiply + uint16_t off_blocks_u, off_mask_u, off_rowin_u, off_tiles_u, off_mtiles_u, off_scratch_u; + uint16_t rowin_units, tile_units, off_extra_units, off_score_units; + union { + struct { + uint32_t total_rows; + uint32_t rs_top, rs_extra, rs_score, rs_dst; // flat row strides (bytes of row 1) + uint32_t mask_nb1; // mask row stride, rows of the mask are dense above it + union { + uint32_t mask_bytes; // resident mask (1 row) + struct { // broadcast mask: row r -> (i1, i2, i3) -> mask row (i mod mne) + uint32_t mp[5]; // fastdiv multipliers of dst ne1, ne2 and mask ne1, ne2, ne3 + uint16_t ne[5]; // the divisors + uint8_t l[5]; // fastdiv shifts + } dec; + } m; + } rows; + struct { + uint32_t picks_per_thread; // per virtual worker, multiple of the lanes of one vector + uint16_t blocks_slice_u, top_slice_u, extra_slice_u, score_slice_u, flags_slice_u; // per virtual worker slices + } picks; + } u; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_top_k_mask_kernel_params) <= 128, "htp_top_k_mask_kernel_params is too large for kernel_params blob"); +#endif + +#endif // HTP_TOP_K_MASK_OPS_H diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index ae910f294b4e..1630ff52c138 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10249,6 +10249,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 16, 0, false, 32)); test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 64, 0, false, 7, 1, 3)); test_cases.emplace_back(new test_top_k_mask(type, 4096, 0, 0, 2048, 0, false, 5)); + // one row, no blocks, fewer than 128 picks: no read past the pick list + test_cases.emplace_back(new test_top_k_mask(type, 1001, 0, 0, 16, 3, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 256, 0, 0, 40, 0, true, 1)); + // one row (single-row kernel) with each optional input combination: no score and no extra, blocks only, blocks and extra without score + test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 16, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 0, false, 1)); + test_cases.emplace_back(new test_top_k_mask(type, 1024, 4, 200, 32, 5, false, 1)); // cells with scores and extra cells test_cases.emplace_back(new test_top_k_mask(type, 512, 0, 0, 32, 5, true, 9)); // blocks picked per head, the mask broadcast over the heads From b97627967c89a5ef135c2da945a036abe2d22ed3 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 3 Nov 2026 23:47:11 +0100 Subject: [PATCH 035/286] model, mtmd: add chatterbox TTS support (#30993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add chatterbox TTS support Co-authored-by: Pascal * add built-in voice support * rm redundant code * clean up * clean up 2 * Apply batched suggestions from code review Co-authored-by: Sigbjørn Skjæret * improve --------- Co-authored-by: Pascal Co-authored-by: Sigbjørn Skjæret --- conversion/__init__.py | 2 + conversion/base.py | 12 +- conversion/chatterbox.py | 471 ++++++++++++++++ gguf-py/gguf/constants.py | 2 + src/models/gpt2.cpp | 4 + tools/mtmd/CMakeLists.txt | 2 + tools/mtmd/clip-impl.h | 4 + tools/mtmd/clip-model.h | 237 +++++++- tools/mtmd/clip.cpp | 714 +++++++++++++++++++++++- tools/mtmd/clip.h | 32 +- tools/mtmd/models/chatterbox-gen.cpp | 542 ++++++++++++++++++ tools/mtmd/models/chatterbox-spkenc.cpp | 156 ++++++ tools/mtmd/models/models.h | 47 ++ tools/mtmd/mtmd-audio.cpp | 475 ++++++++++++++++ tools/mtmd/mtmd-audio.h | 20 + tools/mtmd/mtmd-helper-gen.cpp | 493 +++++++++++++--- tools/mtmd/mtmd.cpp | 473 +++++++++++++++- tools/mtmd/mtmd.h | 7 + tools/tts/README.md | 33 ++ tools/tts/tts.cpp | 5 + 20 files changed, 3648 insertions(+), 83 deletions(-) create mode 100644 conversion/chatterbox.py create mode 100644 tools/mtmd/models/chatterbox-gen.cpp create mode 100644 tools/mtmd/models/chatterbox-spkenc.cpp diff --git a/conversion/__init__.py b/conversion/__init__.py index 6cea1ec7890c..1370dd3b1ffc 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -45,6 +45,7 @@ "ChameleonForConditionalGeneration": "chameleon", "ChatGLMForConditionalGeneration": "chatglm", "ChatGLMModel": "chatglm", + "ChatterboxModel": "chatterbox", "ClefModel": "clef", "CloverMoeModelLM": "clover", "CodeShellForCausalLM": "codeshell", @@ -330,6 +331,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "AudioFlamingo3ForConditionalGeneration": "ultravox", + "ChatterboxModel": "chatterbox", "ClefModel": "clef", "CogVLMForCausalLM": "cogvlm", "Cohere2VisionForConditionalGeneration": "command_r", diff --git a/conversion/base.py b/conversion/base.py index 9d9d6e875adf..28f2bf241d59 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -225,8 +225,6 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call tensors: dict[str, Callable[[], Tensor]] = {} if remote_hf_model_id is not None: - is_safetensors = True - logger.info(f"Using remote model with HuggingFace id: {remote_hf_model_id}") remote_tensors = gguf.utility.SafetensorRemote.get_list_tensors_hf_model(remote_hf_model_id) for name, remote_tensor in remote_tensors.items(): @@ -237,9 +235,13 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call return tensors - prefix = "model" if not self.is_mistral_format else "consolidated" + return self.index_local_tensors("model" if not self.is_mistral_format else "consolidated") + + # index the local "*.safetensors" files (or pytorch_model*.bin), through filter_tensors + def index_local_tensors(self, prefix: str) -> dict[str, Callable[[], Tensor]]: + tensors: dict[str, Callable[[], Tensor]] = {} part_names: list[str] = ModelBase.get_model_part_names(self.dir_model, prefix, ".safetensors") - is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / "model.safetensors.index.json").is_file()) + is_safetensors: bool = len(part_names) > 0 or (not self.is_mistral_format and (self.dir_model / f"{prefix}.safetensors.index.json").is_file()) if not is_safetensors: part_names = ModelBase.get_model_part_names(self.dir_model, "pytorch_model", ".bin") @@ -247,7 +249,7 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call tensor_names_from_parts: set[str] = set() if not self.is_mistral_format: - index_name = "model.safetensors" if is_safetensors else "pytorch_model.bin" + index_name = f"{prefix}.safetensors" if is_safetensors else "pytorch_model.bin" index_name += ".index.json" index_file = self.dir_model / index_name diff --git a/conversion/chatterbox.py b/conversion/chatterbox.py new file mode 100644 index 000000000000..4ab93860c506 --- /dev/null +++ b/conversion/chatterbox.py @@ -0,0 +1,471 @@ +# Chatterbox (ResembleAI) turbo and multilingual v3, from the raw checkpoints of the official repos +# the checkpoints ship no config.json, the variant is detected from the talker file and the hparams come from the reference code +# talker: text_emb and speech_emb are fused into one [text | speech] vocab, speech token i is <|speech_i|> +# mmproj: the s3gen sidecar, the voice and conditioning encoders, the position and speech tables, and the conds.pt voice + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Iterable + +import torch +import torch.nn.functional as F + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, TextModel, MmprojModel, LazyTorchTensor, gguf, logger +from .llama import LlamaModel + +# checkpoint file prefixes, the files are ".safetensors" +TURBO_TALKER = "t3_turbo_v1" +MTL_TALKER = "t3_mtl23ls_v3" +TURBO_S3GEN = "s3gen_meanflow" +MTL_S3GEN = "s3gen_v3" +VE = "ve" + +# relative speech ids shared by both variants (start/stop_speech_token in the reference) +SPEECH_BOS = 6561 +SPEECH_EOS = 6562 + +# sampling defaults of the reference, samplers it does not use are disabled so the common defaults do not apply +TURBO_TOP_K = 1000 +MTL_TOP_K = 0 +TURBO_SAMPLING = {"min_p": 0.0, "top_p": 0.95, "temp": 0.8, "penalty_repeat": 1.2} +MTL_SAMPLING = {"min_p": 0.05, "top_p": 1.0, "temp": 0.8, "penalty_repeat": 1.2} +# the reference repetition penalty covers the whole generated history +PENALTY_LAST_N = -1 + + +def _has_file(dir_model: Path, prefix: str) -> bool: + return (dir_model / f"{prefix}.safetensors").is_file() + + +def _is_turbo(dir_model: Path) -> bool: + if _has_file(dir_model, TURBO_TALKER): + return True + if _has_file(dir_model, MTL_TALKER): + return False + raise FileNotFoundError(f"no chatterbox talker checkpoint in {dir_model}") + + +# talker hparams of the reference (GPT2_MEDIUM_CONFIG / LLAMA_520M_CONFIG_DICT, T3Config) +TURBO_TEXT_CONFIG = { + "n_layer": 24, "n_embd": 1024, "n_inner": 4096, "n_head": 16, "n_ctx": 8196, "layer_norm_epsilon": 1e-05, + "vocab_size": 50276, "speech_vocab_size": 6563, +} +MTL_TEXT_CONFIG = { + "num_hidden_layers": 30, "hidden_size": 1024, "intermediate_size": 4096, "num_attention_heads": 16, + "num_key_value_heads": 16, "head_dim": 64, "max_position_embeddings": 131072, "rms_norm_eps": 1e-05, + "rope_theta": 500000.0, + "rope_scaling": {"factor": 8.0, "high_freq_factor": 4.0, "low_freq_factor": 1.0, "original_max_position_embeddings": 8192}, + "vocab_size": 2454, "speech_vocab_size": 8194, +} + + +def _is_chatterbox_checkpoint(dir_model: Path) -> bool: + return _has_file(dir_model, VE) and (_has_file(dir_model, TURBO_TALKER) or _has_file(dir_model, MTL_TALKER)) + + +@ModelBase.register_hparams_loader(_is_chatterbox_checkpoint) +def _load_chatterbox_hparams(dir_model: Path) -> dict[str, Any]: + turbo = _is_turbo(dir_model) + logger.info(f"gguf: detected Chatterbox {'turbo' if turbo else 'multilingual v3'} checkpoint") + text_config = dict(TURBO_TEXT_CONFIG if turbo else MTL_TEXT_CONFIG) + # the mmproj tensors are named explicitly, they do not go through the block tensor map + return {"architectures": ["ChatterboxModel"], "text_config": text_config, "audio_config": {"num_hidden_layers": 0}} + + +@ModelBase.register("ChatterboxModel") +class ChatterboxTalkerModel(TextModel): + model_arch = gguf.MODEL_ARCH.LLAMA # multilingual; the turbo constructor switches to GPT2 + + def __init__(self, dir_model: Path, *args, **kwargs): + self.is_turbo = _is_turbo(dir_model) + if self.is_turbo: + self.model_arch = gguf.MODEL_ARCH.GPT2 + super().__init__(dir_model, *args, **kwargs) + self._text_embd: Tensor | None = None + self._speech_embd: Tensor | None = None + self._text_head: Tensor | None = None + self._speech_head: Tensor | None = None + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + return self.index_local_tensors(TURBO_TALKER if self.is_turbo else MTL_TALKER) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + # the reference feeds inputs_embeds, the transformer input embeddings are unused + if name in ("tfmr.wte.weight", "tfmr.embed_tokens.weight"): + return None + if name.startswith(("cond_enc.", "text_pos_emb.", "speech_pos_emb.")): + return None + return super().filter_tensors(item) + + def set_vocab(self): + if self.is_turbo: + self._set_vocab_turbo() + else: + self._set_vocab_mtl() + + def _speech_token_names(self, n_speech: int) -> list[str]: + return [f"<|speech_{i}|>" for i in range(n_speech)] + + def _set_vocab_turbo(self): + # stock GPT-2 BPE from the checkpoint dir, extended with the speech tokens + tokens, toktypes, tokpre = self.get_vocab_base() + n_text = len(tokens) + n_speech = self.hparams["speech_vocab_size"] + speech = self._speech_token_names(n_speech) + tokens += speech + toktypes += [gguf.TokenType.CONTROL] * n_speech + + with open(self.dir_model / "merges.txt", "r", encoding="utf-8") as f: + merges = [line.rstrip("\n") for line in f if line.strip() and not line.startswith("#version")] + + self._write_vocab(tokens, toktypes, merges, tokpre, n_text) + + def _write_vocab(self, tokens: list[str], toktypes: list[int], merges: list[str], tokpre: str, n_text: int): + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_token_merges(merges) + + self.gguf_writer.add_bos_token_id(n_text + SPEECH_BOS) + self.gguf_writer.add_eos_token_id(n_text + SPEECH_EOS) + self.gguf_writer.add_add_bos_token(False) + self.gguf_writer.add_add_eos_token(False) + + # the reference samples the speech head only + self.gguf_writer.add_suppress_tokens(list(range(n_text))) + + def _set_vocab_mtl(self): + # the reference BPE is char-level, it is re-encoded to byte-level with extra merges that rebuild each multi-byte char + # its lowercasing is folded into the embedding table, so the prompt needs no text preprocessing + with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f: + tok = json.load(f) + + byte_map = gguf.vocab.bytes_to_unicode() + + def enc(s: str) -> str: + return "".join(byte_map[b] for b in s.encode("utf-8")) + + n_text = self.hparams["vocab_size"] + tokens: list[str] = [f"[unused_{i}]" for i in range(n_text)] + toktypes = [int(gguf.TokenType.UNUSED)] * n_text + char_merges: list[str] = [] + for t, i in tok["model"]["vocab"].items(): + if t == " ": + # the reference substitutes [SPACE] before encoding, the raw space token is never used + continue + tokens[i] = enc(t) + toktypes[i] = int(gguf.TokenType.NORMAL) + if len(t) == 1 and len(t.encode("utf-8")) > 1: + parts = [byte_map[b] for b in t.encode("utf-8")] + for k in range(1, len(parts)): + char_merges.append("".join(parts[:k]) + " " + parts[k]) + for entry in tok.get("added_tokens", []): + tokens[entry["id"]] = entry["content"] + toktypes[entry["id"]] = int(gguf.TokenType.CONTROL) + + # raw spaces tokenize to [SPACE] directly + space_id = tok["model"]["vocab"]["[SPACE]"] + tokens[space_id] = enc(" ") + toktypes[space_id] = int(gguf.TokenType.NORMAL) + + n_speech = self.hparams["speech_vocab_size"] + tokens += self._speech_token_names(n_speech) + toktypes += [int(gguf.TokenType.CONTROL)] * n_speech + + # chars are atomic in the reference, they must form before any of its merges + merges = char_merges + for m in tok["model"].get("merges", []): + a, b = m if isinstance(m, list) else m.split(" ") + merges.append(enc(a) + " " + enc(b)) + + self._write_vocab(tokens, toktypes, merges, "default", n_text) + + def _mtl_case_fold_pairs(self) -> list[tuple[int, int]]: + # (upper_id, lower_id) of the single-char vocab entries, for every script in the vocab + with open(self.dir_model / "mtl_tokenizer.json", "r", encoding="utf-8") as f: + vocab = json.load(f)["model"]["vocab"] + pairs: list[tuple[int, int]] = [] + for t, i in vocab.items(): + if len(t) != 1: + continue + low = t.lower() + if low != t and len(low) == 1 and low in vocab: + pairs.append((i, vocab[low])) + return pairs + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + sampling = TURBO_SAMPLING if self.is_turbo else MTL_SAMPLING + self.gguf_writer.add_sampling_top_k(TURBO_TOP_K if self.is_turbo else MTL_TOP_K) + self.gguf_writer.add_sampling_min_p(sampling["min_p"]) + self.gguf_writer.add_sampling_top_p(sampling["top_p"]) + self.gguf_writer.add_sampling_temp(sampling["temp"]) + self.gguf_writer.add_sampling_penalty_repeat(sampling["penalty_repeat"]) + self.gguf_writer.add_sampling_penalty_last_n(PENALTY_LAST_N) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if self.is_turbo: + return + # llama3 rope scaling baked into the rope_freqs factors tensor + rp = self.hparams["rope_scaling"] + dim = self.hparams["head_dim"] + base = self.hparams["rope_theta"] + freqs = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) + factor = rp["factor"] + low_freq_wavelen = rp["original_max_position_embeddings"] / rp["low_freq_factor"] + high_freq_wavelen = rp["original_max_position_embeddings"] / rp["high_freq_factor"] + rope_factors = [] + for freq in freqs: + wavelen = 2 * math.pi / freq + if wavelen < high_freq_wavelen: + rope_factors.append(1) + elif wavelen > low_freq_wavelen: + rope_factors.append(factor) + else: + smooth = (rp["original_max_position_embeddings"] / wavelen - rp["low_freq_factor"]) / (rp["high_freq_factor"] - rp["low_freq_factor"]) + rope_factors.append(1 / ((1 - smooth) / factor + smooth)) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) + + def _maybe_emit_fused(self) -> Iterable[tuple[str, Tensor]]: + if self._text_embd is not None and self._speech_embd is not None: + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), + torch.cat([self._text_embd, self._speech_embd], dim=0)) + self._text_embd = None + self._speech_embd = None + if self._text_head is not None and self._speech_head is not None: + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), + torch.cat([self._text_head, self._speech_head], dim=0)) + self._text_head = None + self._speech_head = None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # generate_extra_tensors output comes back through here with its final name + if name.startswith("rope_freqs"): + yield (name, data_torch) + return + # fused [text | speech] vocab: embeddings and output head + if name == "text_emb.weight": + if not self.is_turbo: + # the reference lowercases its input, the uppercase rows take the lowercase ones + fold = list(range(data_torch.shape[0])) + for upper, lower in self._mtl_case_fold_pairs(): + fold[upper] = lower + data_torch = data_torch[torch.tensor(fold)] + self._text_embd = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_emb.weight": + self._speech_embd = data_torch + yield from self._maybe_emit_fused() + return + if name == "text_head.weight": + self._text_head = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_head.weight": + self._speech_head = data_torch + yield from self._maybe_emit_fused() + return + if name == "speech_head.bias": + # the text head has no bias, its zone is suppressed at sampling anyway + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT, suffix=".bias"), F.pad(data_torch, (self.hparams["vocab_size"], 0))) + return + + assert name.startswith("tfmr.") + name = name[len("tfmr."):] + + if self.is_turbo: + # HF GPT-2 layout: Conv1D style weights are stored transposed + if name.endswith((".c_attn.weight", ".c_proj.weight", ".c_fc.weight")): + data_torch = data_torch.transpose(1, 0) + yield (self.map_tensor_name(name), data_torch) + return + + if name.endswith("q_proj.weight"): + data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], None) + if name.endswith("k_proj.weight"): + data_torch = LlamaModel.permute(data_torch, self.hparams["num_attention_heads"], self.hparams["num_key_value_heads"]) + yield (self.map_tensor_name("model." + name), data_torch) + + +@ModelBase.register("ChatterboxModel") +class ChatterboxMmprojModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + def __init__(self, dir_model: Path, *args, **kwargs): + self.is_turbo = _is_turbo(dir_model) + super().__init__(dir_model, *args, **kwargs) + self._wnorm_g: dict[str, Tensor] = {} + self._wnorm_v: dict[str, Tensor] = {} + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("audio_config") + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + tensors: dict[str, Callable[[], Tensor]] = {} + for prefix in (TURBO_S3GEN if self.is_turbo else MTL_S3GEN, VE, TURBO_TALKER if self.is_turbo else MTL_TALKER): + tensors.update(self.index_local_tensors(prefix)) + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + # the s3gen, ve and talker checkpoints have disjoint top-level names + name, gen = item + new_name: str | None = None + # batchnorm bookkeeping (over the gguf name length cap) and the tokenizer dsp buffers are unused + if name.endswith("num_batches_tracked") or name in ("tokenizer.window", "tokenizer._mel_filters"): + return None + for src, dst in ( + ("flow.encoder.", "a.gen.fenc."), + ("flow.decoder.estimator.", "a.gen.est."), + ("mel2wav.", "a.gen.hift."), + ("speaker_encoder.", "a.spk."), + ("tokenizer.", "a.s3tok."), + ("flow.", "a.gen.flow."), # input_embedding, encoder_proj, spk_embed_affine_layer + ("lstm.", "a.ve.lstm."), + ("proj.", "a.ve.proj."), + ("cond_enc.", "a.cenc."), + ): + if name.startswith(src): + new_name = dst + name[len(src):] + break + if name == "text_pos_emb.emb.weight": + new_name = "a.gen.t3.text_pos_emb" + elif name == "speech_pos_emb.emb.weight": + new_name = "a.gen.t3.speech_pos_emb" + elif name == "speech_emb.weight": + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD] + ".weight" + if new_name is None: + return None + return super().filter_tensors((new_name, gen)) + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + + # speaker encoder, the front-end hparams are hard-coded in clip.cpp + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX_SPKENC) + self.gguf_writer.add_audio_projection_dim(80) + self.gguf_writer.add_audio_num_mel_bins(80) + self.gguf_writer.add_audio_block_count(0) + self.gguf_writer.add_audio_embedding_length(192) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(192) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + # audio generator (s3gen, chatterbox projector); the flow encoder shape + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.CHATTERBOX) + self.gguf_writer.add_gen_audio_projection_dim(80) + self.gguf_writer.add_gen_audio_embedding_length(512) + self.gguf_writer.add_gen_audio_feed_forward_length(2048) + self.gguf_writer.add_gen_audio_block_count(6) + self.gguf_writer.add_gen_audio_head_count(8) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5) + + def _fuse_weight_norm(self, name: str, data_torch: Tensor) -> tuple[str, Tensor] | None: + # torch weight_norm parametrization: weight = g * v / |v| over dims 1..n + base = name.split(".parametrizations.weight.original")[0] + if name.endswith("original0"): + self._wnorm_g[base] = data_torch + else: + self._wnorm_v[base] = data_torch + if base in self._wnorm_g and base in self._wnorm_v: + g = self._wnorm_g.pop(base) + v = self._wnorm_v.pop(base) + norm = v.float().norm(dim=tuple(range(1, v.dim())), keepdim=True) + return (base + ".weight", g.float() * v.float() / norm) + return None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if ".parametrizations.weight.original" in name: + fused = self._fuse_weight_norm(name, data_torch) + if fused is not None: + yield fused + return + yield (name, data_torch) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + del name, bid + # tensors read on the host stay F32 + if new_name.startswith(("a.ve.", "a.cenc.", "voice.", "a.gen.t3.", "a.gen.hift.m_source.")): + return gguf.GGMLQuantizationType.F32 + # the conv ops have no BF16 kernels + if n_dims >= 2 and new_name.endswith((".weight", ".weight_v")): + return gguf.GGMLQuantizationType.F16 + return gguf.GGMLQuantizationType.F32 + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + conds = torch.load(self.dir_model / "conds.pt", map_location="cpu", weights_only=True) + t3c = conds["t3"] if isinstance(conds, dict) else conds.t3 + genc = conds["gen"] if isinstance(conds, dict) else conds.gen + t3c = vars(t3c) if not isinstance(t3c, dict) else t3c + genc = vars(genc) if not isinstance(genc, dict) else genc + + def talker_tensor(name: str) -> Tensor: + return LazyTorchTensor.to_eager(self.model_tensors[name]()) + + # the conds.pt voice: voice.default holds the talker conditioning rows, voice.default.ref_* the flow decoder reference + # 1D tensors are written as F32, the speech ids stay exact + yield ("voice.default.ref_tokens", genc["prompt_token"][0].float()) + yield ("voice.default.ref_feat", genc["prompt_feat"][0].float()) + yield ("voice.default.ref_spk", genc["embedding"][0].float()) + + spkr_w = talker_tensor("a.cenc.spkr_enc.weight").float() + spkr_b = talker_tensor("a.cenc.spkr_enc.bias").float() + spkr_row = spkr_w @ t3c["speaker_emb"][0].float() + spkr_b + speech_emb = talker_tensor(self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD)).float() + cond_tokens = t3c["cond_prompt_speech_tokens"][0] + + if self.is_turbo: + # turbo talker conditioning: [spkr, speech rows of the cond tokens] + yield ("voice.default", torch.cat([spkr_row[None], speech_emb[cond_tokens]], dim=0)) + return + + # multilingual talker conditioning [spkr, perceiver x32, emotion], from the reference perceiver over the cond speech tokens + with torch.no_grad(): + speech_pos = talker_tensor("a.gen.t3.speech_pos_emb").float() + pse = speech_emb[cond_tokens] + speech_pos[: cond_tokens.shape[0]] + + ln_w = talker_tensor("a.cenc.perceiver.attn.norm.weight").float() + ln_b = talker_tensor("a.cenc.perceiver.attn.norm.bias").float() + wq = talker_tensor("a.cenc.perceiver.attn.to_q.weight").float() + bq = talker_tensor("a.cenc.perceiver.attn.to_q.bias").float() + wk = talker_tensor("a.cenc.perceiver.attn.to_k.weight").float() + bk = talker_tensor("a.cenc.perceiver.attn.to_k.bias").float() + wv = talker_tensor("a.cenc.perceiver.attn.to_v.weight").float() + bv = talker_tensor("a.cenc.perceiver.attn.to_v.bias").float() + wo = talker_tensor("a.cenc.perceiver.attn.proj_out.weight").float() + bo = talker_tensor("a.cenc.perceiver.attn.proj_out.bias").float() + query = talker_tensor("a.cenc.perceiver.pre_attention_query")[0].float() + + n_head = 4 + n_e = query.shape[1] + + def attn_block(x1: Tensor, x2: Tensor) -> Tensor: + nx1 = F.layer_norm(x1, (n_e,), ln_w, ln_b) + nx2 = F.layer_norm(x2, (n_e,), ln_w, ln_b) + q = (nx1 @ wq.T + bq).view(-1, n_head, n_e // n_head).transpose(0, 1) + k = (nx2 @ wk.T + bk).view(-1, n_head, n_e // n_head).transpose(0, 1) + v = (nx2 @ wv.T + bv).view(-1, n_head, n_e // n_head).transpose(0, 1) + ctx = F.scaled_dot_product_attention(q, k, v) + ctx = ctx.transpose(0, 1).reshape(-1, n_e) + return ctx @ wo.T + bo + x1 + + pre = attn_block(query, pse) + p32 = attn_block(pre, pre) + + emo = talker_tensor("a.cenc.emotion_adv_fc.weight").float() + emo_row = emo[:, 0] * t3c["emotion_adv"].reshape(-1)[0] + yield ("voice.default", torch.cat([spkr_row[None], p32, emo_row[None]], dim=0)) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index c4c4a4c65eda..fcd17f4b5068 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6916,6 +6916,8 @@ class VisionProjectorType: QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor POCKETTTS_SPKENC = "pockettts_spkenc" # audio: mimi encoder as voice-prompt encoder POCKETTTS_GEN = "pockettts_gen" # audio generation: flow-matching decoder + mimi decoder + CHATTERBOX_SPKENC = "chatterbox_spkenc" # audio: CAMPPlus speaker encoder + CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder SOPRANO = "soprano" # audio generation: Vocos decoder HUNYUANVL = "hunyuanvl" PARAKEET = "parakeet" # audio diff --git a/src/models/gpt2.cpp b/src/models/gpt2.cpp index d6ca64744f49..102dad97e3b3 100644 --- a/src/models/gpt2.cpp +++ b/src/models/gpt2.cpp @@ -22,6 +22,7 @@ void llama_model_gpt2::load_arch_tensors(llama_model_loader &) { output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"), {n_embd}, 0); output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); + output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"), {n_vocab}, TENSOR_NOT_REQUIRED); // if output is NULL, init from the input tok embed if (output == NULL) { @@ -140,6 +141,9 @@ llama_model_gpt2::graph::graph(const llama_model & model, const llm_graph_params res->t_embd = cur; cur = build_lora_mm(model.output, cur, model.output_s); + if (model.output_b) { + cur = ggml_add(ctx0, cur, model.output_b); + } cb(cur, "result_output", -1); res->t_logits = cur; diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index e514de60bdf9..1b178111d132 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -28,6 +28,8 @@ add_library(mtmd clip-model.h clip-graph.h models/models.h + models/chatterbox-gen.cpp + models/chatterbox-spkenc.cpp models/cogvlm.cpp models/conformer.cpp models/deepseek4v.cpp diff --git a/tools/mtmd/clip-impl.h b/tools/mtmd/clip-impl.h index 52d5fffe33f7..dcd6ec2162a4 100644 --- a/tools/mtmd/clip-impl.h +++ b/tools/mtmd/clip-impl.h @@ -599,6 +599,8 @@ enum projector_type { PROJECTOR_TYPE_SENSENOVA_U1, PROJECTOR_TYPE_COHERE2V, PROJECTOR_TYPE_MUSE_SPARK_AUDIO, + PROJECTOR_TYPE_CHATTERBOX, + PROJECTOR_TYPE_CHATTERBOX_SPKENC, PROJECTOR_TYPE_UNKNOWN, }; @@ -681,6 +683,8 @@ static std::map PROJECTOR_TYPE_NAMES = { { PROJECTOR_TYPE_SENSENOVA_U1, "sensenova_u1"}, { PROJECTOR_TYPE_COHERE2V, "cohere2v"}, { PROJECTOR_TYPE_MUSE_SPARK_AUDIO, "muse-spark-audio"}, + { PROJECTOR_TYPE_CHATTERBOX, "chatterbox"}, + { PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"}, }; static projector_type clip_projector_type_from_string(const std::string & str) { diff --git a/tools/mtmd/clip-model.h b/tools/mtmd/clip-model.h index c42f417c83ca..b5a4a1225bd8 100644 --- a/tools/mtmd/clip-model.h +++ b/tools/mtmd/clip-model.h @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -611,6 +612,235 @@ struct clip_code2wav { ggml_tensor * dac_post_conv_b = nullptr; }; +struct clip_chatterbox { + // espnet rel-pos conformer layer of the flow encoder + struct enc_layer { + ggml_tensor * norm_mha_w = nullptr; + ggml_tensor * norm_mha_b = nullptr; + ggml_tensor * attn_q_w = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_k_w = nullptr; + ggml_tensor * attn_k_b = nullptr; + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_v_b = nullptr; + ggml_tensor * attn_pos_w = nullptr; // linear_pos, no bias + ggml_tensor * attn_pos_bias_u = nullptr; + ggml_tensor * attn_pos_bias_v = nullptr; + ggml_tensor * attn_out_w = nullptr; + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * norm_ff_w = nullptr; + ggml_tensor * norm_ff_b = nullptr; + ggml_tensor * ffn_1_w = nullptr; + ggml_tensor * ffn_1_b = nullptr; + ggml_tensor * ffn_2_w = nullptr; + ggml_tensor * ffn_2_b = nullptr; + }; + + // causal conv block of the estimator (conv k3 left padded + layer norm) + struct causal_block { + ggml_tensor * conv_w = nullptr; + ggml_tensor * conv_b = nullptr; + ggml_tensor * norm_w = nullptr; + ggml_tensor * norm_b = nullptr; + }; + + // time conditioned resnet of the estimator + struct resnet { + causal_block block1; + causal_block block2; + ggml_tensor * mlp_w = nullptr; // time projection (mlp.1) + ggml_tensor * mlp_b = nullptr; + ggml_tensor * res_conv_w = nullptr; + ggml_tensor * res_conv_b = nullptr; + }; + + // diffusers style transformer block of the estimator + struct tfm_block { + ggml_tensor * norm1_w = nullptr; + ggml_tensor * norm1_b = nullptr; + ggml_tensor * attn_q_w = nullptr; // no bias on qkv + ggml_tensor * attn_k_w = nullptr; + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_out_w = nullptr; // to_out.0 + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * norm3_w = nullptr; + ggml_tensor * norm3_b = nullptr; + ggml_tensor * ff_in_w = nullptr; // ff.net.0.proj + ggml_tensor * ff_in_b = nullptr; + ggml_tensor * ff_out_w = nullptr; // ff.net.2 + ggml_tensor * ff_out_b = nullptr; + }; + + // one estimator stage: resnet, transformer stack, boundary causal conv (null on mid stages) + struct est_stage { + resnet res; + std::vector tfm; + ggml_tensor * conv_w = nullptr; + ggml_tensor * conv_b = nullptr; + }; + + // snake resblock unit of the hift vocoder + struct hift_res_unit { + ggml_tensor * act1_alpha = nullptr; + ggml_tensor * act2_alpha = nullptr; + ggml_tensor * conv1_w = nullptr; + ggml_tensor * conv1_b = nullptr; + ggml_tensor * conv2_w = nullptr; + ggml_tensor * conv2_b = nullptr; + }; + struct hift_res { + std::vector units; // dilations 1/3/5 on conv1 + }; + + // one hift upsample stage: conv transpose, source injection, 3 resblocks + struct hift_up { + ggml_tensor * up_w = nullptr; + ggml_tensor * up_b = nullptr; + ggml_tensor * source_down_w = nullptr; + ggml_tensor * source_down_b = nullptr; + hift_res source_res; + hift_res res_0; + hift_res res_1; + hift_res res_2; + }; + + // f0 predictor conv of the hift vocoder + struct f0_conv { + ggml_tensor * w = nullptr; + ggml_tensor * b = nullptr; + }; + + // s3 tokenizer attention block (neox rope on q/k, fsmn memory on v) + struct s3tok_block { + ggml_tensor * attn_ln_w = nullptr; + ggml_tensor * attn_ln_b = nullptr; + ggml_tensor * attn_q_w = nullptr; + ggml_tensor * attn_q_b = nullptr; + ggml_tensor * attn_k_w = nullptr; // no bias + ggml_tensor * attn_v_w = nullptr; + ggml_tensor * attn_v_b = nullptr; + ggml_tensor * fsmn_w = nullptr; + ggml_tensor * attn_out_w = nullptr; + ggml_tensor * attn_out_b = nullptr; + ggml_tensor * mlp_ln_w = nullptr; + ggml_tensor * mlp_ln_b = nullptr; + ggml_tensor * mlp_in_w = nullptr; + ggml_tensor * mlp_in_b = nullptr; + ggml_tensor * mlp_out_w = nullptr; + ggml_tensor * mlp_out_b = nullptr; + }; + + // folded batchnorm (w/b stay null on the affine free variant) + struct bn { + ggml_tensor * mean = nullptr; + ggml_tensor * var = nullptr; + ggml_tensor * w = nullptr; + ggml_tensor * b = nullptr; + }; + + // fcm residual 2d block of the speaker encoder, the shortcut is null on stride 1 blocks + struct spk_res2d { + ggml_tensor * conv1_w = nullptr; + bn bn1; + ggml_tensor * conv2_w = nullptr; + bn bn2; + ggml_tensor * shortcut_w = nullptr; + bn shortcut_bn; + }; + + // cam dense tdnn layer of the speaker encoder + struct spk_cam_layer { + bn nl1_bn; + ggml_tensor * linear1_w = nullptr; + bn nl2_bn; + ggml_tensor * local_w = nullptr; // cam_layer.linear_local + ggml_tensor * ctx1_w = nullptr; // cam_layer.linear1 + ggml_tensor * ctx1_b = nullptr; + ggml_tensor * ctx2_w = nullptr; // cam_layer.linear2 + ggml_tensor * ctx2_b = nullptr; + }; + struct spk_cam_block { + std::vector layers; + bn transit_bn; + ggml_tensor * transit_w = nullptr; + }; + + // flow encoder + ggml_tensor * input_embedding_w = nullptr; // flow.input_embedding + ggml_tensor * embed_linear_w = nullptr; // fenc.embed.out.0 + ggml_tensor * embed_linear_b = nullptr; + ggml_tensor * embed_norm_w = nullptr; // fenc.embed.out.1 + ggml_tensor * embed_norm_b = nullptr; + ggml_tensor * pre_conv1_w = nullptr; // pre_lookahead_layer + ggml_tensor * pre_conv1_b = nullptr; + ggml_tensor * pre_conv2_w = nullptr; + ggml_tensor * pre_conv2_b = nullptr; + std::vector enc; + ggml_tensor * up_conv_w = nullptr; // fenc.up_layer.conv + ggml_tensor * up_conv_b = nullptr; + ggml_tensor * up_embed_linear_w = nullptr; // fenc.up_embed.out.0 + ggml_tensor * up_embed_linear_b = nullptr; + ggml_tensor * up_embed_norm_w = nullptr; // fenc.up_embed.out.1 + ggml_tensor * up_embed_norm_b = nullptr; + std::vector up_enc; + ggml_tensor * after_norm_w = nullptr; + ggml_tensor * after_norm_b = nullptr; + ggml_tensor * encoder_proj_w = nullptr; // flow.encoder_proj + ggml_tensor * encoder_proj_b = nullptr; + + // cfm estimator + ggml_tensor * time_mlp_1_w = nullptr; + ggml_tensor * time_mlp_1_b = nullptr; + ggml_tensor * time_mlp_2_w = nullptr; + ggml_tensor * time_mlp_2_b = nullptr; + ggml_tensor * time_embed_mixer_w = nullptr; // meanflow variant only + est_stage est_down; + std::vector est_mid; + est_stage est_up; + causal_block est_final_block; + ggml_tensor * est_final_proj_w = nullptr; + ggml_tensor * est_final_proj_b = nullptr; + + // hift vocoder + ggml_tensor * hift_pre_w = nullptr; + ggml_tensor * hift_pre_b = nullptr; + std::vector hift_ups; + ggml_tensor * hift_post_w = nullptr; + ggml_tensor * hift_post_b = nullptr; + std::vector f0_condnet; + ggml_tensor * f0_classifier_w = nullptr; + ggml_tensor * f0_classifier_b = nullptr; + + // s3 tokenizer + ggml_tensor * s3tok_conv1_w = nullptr; + ggml_tensor * s3tok_conv1_b = nullptr; + ggml_tensor * s3tok_conv2_w = nullptr; + ggml_tensor * s3tok_conv2_b = nullptr; + std::vector s3tok_blocks; + ggml_tensor * s3tok_down_w = nullptr; // quantizer._codebook.project_down + ggml_tensor * s3tok_down_b = nullptr; + + // CAMPPlus speaker encoder + ggml_tensor * spk_conv1_w = nullptr; + bn spk_bn1; + spk_res2d spk_layer1_0; + spk_res2d spk_layer1_1; + spk_res2d spk_layer2_0; + spk_res2d spk_layer2_1; + ggml_tensor * spk_conv2_w = nullptr; + bn spk_bn2; + ggml_tensor * spk_tdnn_w = nullptr; + bn spk_tdnn_bn; + spk_cam_block spk_block1; + spk_cam_block spk_block2; + spk_cam_block spk_block3; + bn spk_out_bn; + ggml_tensor * spk_dense_w = nullptr; + bn spk_dense_bn; + ggml_tensor * spk_affine_w = nullptr; + ggml_tensor * spk_affine_b = nullptr; +}; + struct clip_vocos { ggml_tensor * input_w = nullptr; ggml_tensor * input_b = nullptr; @@ -896,8 +1126,13 @@ struct clip_model { ggml_tensor * gen_upsample_w = nullptr; // depthwise convtr, frame rate -> encoder frame rate std::vector gen_tfm_layers; // mimi decoder_transformer + // chatterbox flow encoder, cfm estimator, hift vocoder, s3 tokenizer and CAMPPlus speaker encoder + clip_chatterbox cbx; + + // chatterbox tensors read on the host, see clip_cbx_read_tensor() + std::map cbx_tensors; // predefined voices, no rows if the voice is named in the prompt (lfm3a) - std::map> gen_voices; + std::map gen_voices; // soprano: Vocos decoder, hidden states -> STFT coefficients clip_vocos vocos; diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index f2c5c31b83c1..e6d931e69f9f 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -1,3 +1,4 @@ +#define _USE_MATH_DEFINES // for M_PI #include "clip.h" #include "clip-impl.h" #include "clip-model.h" @@ -1200,6 +1201,24 @@ static std::unique_ptr clip_get_graph_builder(clip_ctx * ctx, const const int n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1; builder = std::make_unique(ctx, img, gen_process, n_step, n_frames); } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + builder = std::make_unique(ctx, img); + } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; + int n_tokens = params && params->codes ? (int) params->codes->size() : 0; + int n_prompt_mel = 0; + if (n_tokens > 0) { + GGML_ASSERT(params->ref); + n_tokens += (int) params->ref->tokens.size(); + n_prompt_mel = (int) (params->ref->feat.size() / 80); + } + const int vnm = params ? params->vocode_n_mel : 0; + const int vns = params ? params->vocode_n_stft : 0; + builder = std::make_unique(ctx, img, gen_process, n_tokens, n_prompt_mel, vnm, vns); + } break; case PROJECTOR_TYPE_LFM3A_GEN: { const auto gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE; @@ -2184,6 +2203,14 @@ struct clip_model_loader { hparams.audio_window_len = 1024; hparams.audio_hop_len = 256; } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // kaldi fbank front-end of CAMPPlus: 25 ms / 10 ms framing, 512-point spectrum + hparams.audio_sample_rate = 16000; + hparams.audio_n_fft = 512; + hparams.audio_window_len = 400; + hparams.audio_hop_len = 160; + } break; case PROJECTOR_TYPE_LFM3A_GEN: { // fixed by LiquidCodecConfig, head counts come from the tensors @@ -2246,6 +2273,11 @@ struct clip_model_loader { hparams.flow_n_step = 1; hparams.gen_eos_threshold = -4.0f; } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + // s3gen output rate + hparams.audio_sample_rate = 24000; + } break; case PROJECTOR_TYPE_SOPRANO: { // SopranoDecoder defaults, n_fft comes from the head tensor @@ -2649,7 +2681,9 @@ struct clip_model_loader { model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC && model.proj_type != PROJECTOR_TYPE_POCKETTTS_GEN && model.proj_type != PROJECTOR_TYPE_SOPRANO && - model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO); + model.proj_type != PROJECTOR_TYPE_MUSE_SPARK_AUDIO && + model.proj_type != PROJECTOR_TYPE_CHATTERBOX && + model.proj_type != PROJECTOR_TYPE_CHATTERBOX_SPKENC); // layers const int n_layers_to_load = has_standard_layers ? hparams.n_layer : 0; @@ -3537,7 +3571,7 @@ struct clip_model_loader { if (t->type != GGML_TYPE_F32 || ggml_n_dims(t) > 2 || t->ne[0] != model.gen_input_lin_w->ne[1]) { throw std::runtime_error(string_format("%s: invalid voice tensor %s\n", __func__, name)); } - auto & embd = model.gen_voices[name + 6]; + auto & embd = model.gen_voices[name + 6].rows; embd.resize(ggml_nelements(t)); fin.seekg(tensor_offset.at(name), std::ios::beg); if (!fin.read(reinterpret_cast(embd.data()), ggml_nbytes(t))) { @@ -3708,6 +3742,326 @@ struct clip_model_loader { c2w.dac_post_conv_b = get_tensor(string_format(TN_A_GEN_WAV_DAC_POST_CONV, "bias")); } } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + auto & c = model.cbx; + auto has = [&](const std::string & name) { + return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; + }; + auto load_enc_layer = [&](const std::string & p, clip_chatterbox::enc_layer & l) { + l.norm_mha_w = get_tensor(p + ".norm_mha.weight"); + l.norm_mha_b = get_tensor(p + ".norm_mha.bias"); + l.attn_q_w = get_tensor(p + ".self_attn.linear_q.weight"); + l.attn_q_b = get_tensor(p + ".self_attn.linear_q.bias"); + l.attn_k_w = get_tensor(p + ".self_attn.linear_k.weight"); + l.attn_k_b = get_tensor(p + ".self_attn.linear_k.bias"); + l.attn_v_w = get_tensor(p + ".self_attn.linear_v.weight"); + l.attn_v_b = get_tensor(p + ".self_attn.linear_v.bias"); + l.attn_pos_w = get_tensor(p + ".self_attn.linear_pos.weight"); + l.attn_pos_bias_u = get_tensor(p + ".self_attn.pos_bias_u"); + l.attn_pos_bias_v = get_tensor(p + ".self_attn.pos_bias_v"); + l.attn_out_w = get_tensor(p + ".self_attn.linear_out.weight"); + l.attn_out_b = get_tensor(p + ".self_attn.linear_out.bias"); + l.norm_ff_w = get_tensor(p + ".norm_ff.weight"); + l.norm_ff_b = get_tensor(p + ".norm_ff.bias"); + l.ffn_1_w = get_tensor(p + ".feed_forward.w_1.weight"); + l.ffn_1_b = get_tensor(p + ".feed_forward.w_1.bias"); + l.ffn_2_w = get_tensor(p + ".feed_forward.w_2.weight"); + l.ffn_2_b = get_tensor(p + ".feed_forward.w_2.bias"); + }; + auto load_causal = [&](const std::string & p, clip_chatterbox::causal_block & b) { + b.conv_w = get_tensor(p + ".block.0.weight"); + b.conv_b = get_tensor(p + ".block.0.bias"); + b.norm_w = get_tensor(p + ".block.2.weight"); + b.norm_b = get_tensor(p + ".block.2.bias"); + }; + auto load_resnet = [&](const std::string & p, clip_chatterbox::resnet & r) { + load_causal(p + ".block1", r.block1); + load_causal(p + ".block2", r.block2); + r.mlp_w = get_tensor(p + ".mlp.1.weight"); + r.mlp_b = get_tensor(p + ".mlp.1.bias"); + r.res_conv_w = get_tensor(p + ".res_conv.weight"); + r.res_conv_b = get_tensor(p + ".res_conv.bias"); + }; + auto load_tfm = [&](const std::string & p, clip_chatterbox::tfm_block & b) { + b.norm1_w = get_tensor(p + ".norm1.weight"); + b.norm1_b = get_tensor(p + ".norm1.bias"); + b.attn_q_w = get_tensor(p + ".attn1.to_q.weight"); + b.attn_k_w = get_tensor(p + ".attn1.to_k.weight"); + b.attn_v_w = get_tensor(p + ".attn1.to_v.weight"); + b.attn_out_w = get_tensor(p + ".attn1.to_out.0.weight"); + b.attn_out_b = get_tensor(p + ".attn1.to_out.0.bias"); + b.norm3_w = get_tensor(p + ".norm3.weight"); + b.norm3_b = get_tensor(p + ".norm3.bias"); + b.ff_in_w = get_tensor(p + ".ff.net.0.proj.weight"); + b.ff_in_b = get_tensor(p + ".ff.net.0.proj.bias"); + b.ff_out_w = get_tensor(p + ".ff.net.2.weight"); + b.ff_out_b = get_tensor(p + ".ff.net.2.bias"); + }; + auto load_stage = [&](const std::string & p, clip_chatterbox::est_stage & s, bool boundary) { + load_resnet(p + ".0", s.res); + for (int j = 0; has(p + ".1." + std::to_string(j) + ".norm1.weight"); j++) { + clip_chatterbox::tfm_block b; + load_tfm(p + ".1." + std::to_string(j), b); + s.tfm.push_back(b); + } + if (boundary) { + s.conv_w = get_tensor(p + ".2.weight"); + s.conv_b = get_tensor(p + ".2.bias"); + } + }; + auto load_hres = [&](const std::string & p, clip_chatterbox::hift_res & r) { + for (int j = 0; has(p + ".convs1." + std::to_string(j) + ".weight"); j++) { + const std::string js = std::to_string(j); + clip_chatterbox::hift_res_unit u; + u.act1_alpha = get_tensor(p + ".activations1." + js + ".alpha"); + u.act2_alpha = get_tensor(p + ".activations2." + js + ".alpha"); + u.conv1_w = get_tensor(p + ".convs1." + js + ".weight"); + u.conv1_b = get_tensor(p + ".convs1." + js + ".bias"); + u.conv2_w = get_tensor(p + ".convs2." + js + ".weight"); + u.conv2_b = get_tensor(p + ".convs2." + js + ".bias"); + r.units.push_back(u); + } + }; + + // flow encoder + c.input_embedding_w = get_tensor("a.gen.flow.input_embedding.weight"); + c.spk_affine_w = get_tensor("a.gen.flow.spk_embed_affine_layer.weight"); + c.spk_affine_b = get_tensor("a.gen.flow.spk_embed_affine_layer.bias"); + c.embed_linear_w = get_tensor("a.gen.fenc.embed.out.0.weight"); + c.embed_linear_b = get_tensor("a.gen.fenc.embed.out.0.bias"); + c.embed_norm_w = get_tensor("a.gen.fenc.embed.out.1.weight"); + c.embed_norm_b = get_tensor("a.gen.fenc.embed.out.1.bias"); + c.pre_conv1_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.weight"); + c.pre_conv1_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv1.bias"); + c.pre_conv2_w = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.weight"); + c.pre_conv2_b = get_tensor("a.gen.fenc.pre_lookahead_layer.conv2.bias"); + for (int i = 0; has("a.gen.fenc.encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { + clip_chatterbox::enc_layer l; + load_enc_layer("a.gen.fenc.encoders." + std::to_string(i), l); + c.enc.push_back(l); + } + c.up_conv_w = get_tensor("a.gen.fenc.up_layer.conv.weight"); + c.up_conv_b = get_tensor("a.gen.fenc.up_layer.conv.bias"); + c.up_embed_linear_w = get_tensor("a.gen.fenc.up_embed.out.0.weight"); + c.up_embed_linear_b = get_tensor("a.gen.fenc.up_embed.out.0.bias"); + c.up_embed_norm_w = get_tensor("a.gen.fenc.up_embed.out.1.weight"); + c.up_embed_norm_b = get_tensor("a.gen.fenc.up_embed.out.1.bias"); + for (int i = 0; has("a.gen.fenc.up_encoders." + std::to_string(i) + ".norm_mha.weight"); i++) { + clip_chatterbox::enc_layer l; + load_enc_layer("a.gen.fenc.up_encoders." + std::to_string(i), l); + c.up_enc.push_back(l); + } + c.after_norm_w = get_tensor("a.gen.fenc.after_norm.weight"); + c.after_norm_b = get_tensor("a.gen.fenc.after_norm.bias"); + c.encoder_proj_w = get_tensor("a.gen.flow.encoder_proj.weight"); + c.encoder_proj_b = get_tensor("a.gen.flow.encoder_proj.bias"); + + // cfm estimator + c.time_mlp_1_w = get_tensor("a.gen.est.time_mlp.linear_1.weight"); + c.time_mlp_1_b = get_tensor("a.gen.est.time_mlp.linear_1.bias"); + c.time_mlp_2_w = get_tensor("a.gen.est.time_mlp.linear_2.weight"); + c.time_mlp_2_b = get_tensor("a.gen.est.time_mlp.linear_2.bias"); + c.time_embed_mixer_w = get_tensor("a.gen.est.time_embed_mixer.weight", false); + load_stage("a.gen.est.down_blocks.0", c.est_down, true); + for (int i = 0; has("a.gen.est.mid_blocks." + std::to_string(i) + ".0.block1.block.0.weight"); i++) { + clip_chatterbox::est_stage s; + load_stage("a.gen.est.mid_blocks." + std::to_string(i), s, false); + c.est_mid.push_back(std::move(s)); + } + load_stage("a.gen.est.up_blocks.0", c.est_up, true); + load_causal("a.gen.est.final_block", c.est_final_block); + c.est_final_proj_w = get_tensor("a.gen.est.final_proj.weight"); + c.est_final_proj_b = get_tensor("a.gen.est.final_proj.bias"); + + // hift vocoder + c.hift_pre_w = get_tensor("a.gen.hift.conv_pre.weight"); + c.hift_pre_b = get_tensor("a.gen.hift.conv_pre.bias"); + c.hift_post_w = get_tensor("a.gen.hift.conv_post.weight"); + c.hift_post_b = get_tensor("a.gen.hift.conv_post.bias"); + for (int i = 0; has("a.gen.hift.ups." + std::to_string(i) + ".weight"); i++) { + const std::string is = std::to_string(i); + clip_chatterbox::hift_up up; + up.up_w = get_tensor("a.gen.hift.ups." + is + ".weight"); + up.up_b = get_tensor("a.gen.hift.ups." + is + ".bias"); + up.source_down_w = get_tensor("a.gen.hift.source_downs." + is + ".weight"); + up.source_down_b = get_tensor("a.gen.hift.source_downs." + is + ".bias"); + load_hres("a.gen.hift.source_resblocks." + is, up.source_res); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i), up.res_0); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 1), up.res_1); + load_hres("a.gen.hift.resblocks." + std::to_string(3 * i + 2), up.res_2); + c.hift_ups.push_back(std::move(up)); + } + for (int i = 0; has("a.gen.hift.f0_predictor.condnet." + std::to_string(i) + ".weight"); i += 2) { + const std::string is = std::to_string(i); + clip_chatterbox::f0_conv fc; + fc.w = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".weight"); + fc.b = get_tensor("a.gen.hift.f0_predictor.condnet." + is + ".bias"); + c.f0_condnet.push_back(fc); + } + c.f0_classifier_w = get_tensor("a.gen.hift.f0_predictor.classifier.weight"); + c.f0_classifier_b = get_tensor("a.gen.hift.f0_predictor.classifier.bias"); + + // s3 tokenizer + c.s3tok_conv1_w = get_tensor("a.s3tok.encoder.conv1.weight"); + c.s3tok_conv1_b = get_tensor("a.s3tok.encoder.conv1.bias"); + c.s3tok_conv2_w = get_tensor("a.s3tok.encoder.conv2.weight"); + c.s3tok_conv2_b = get_tensor("a.s3tok.encoder.conv2.bias"); + for (int i = 0; has("a.s3tok.encoder.blocks." + std::to_string(i) + ".attn_ln.weight"); i++) { + const std::string p = "a.s3tok.encoder.blocks." + std::to_string(i); + clip_chatterbox::s3tok_block b; + b.attn_ln_w = get_tensor(p + ".attn_ln.weight"); + b.attn_ln_b = get_tensor(p + ".attn_ln.bias"); + b.attn_q_w = get_tensor(p + ".attn.query.weight"); + b.attn_q_b = get_tensor(p + ".attn.query.bias"); + b.attn_k_w = get_tensor(p + ".attn.key.weight"); + b.attn_v_w = get_tensor(p + ".attn.value.weight"); + b.attn_v_b = get_tensor(p + ".attn.value.bias"); + b.fsmn_w = get_tensor(p + ".attn.fsmn_block.weight"); + b.attn_out_w = get_tensor(p + ".attn.out.weight"); + b.attn_out_b = get_tensor(p + ".attn.out.bias"); + b.mlp_ln_w = get_tensor(p + ".mlp_ln.weight"); + b.mlp_ln_b = get_tensor(p + ".mlp_ln.bias"); + b.mlp_in_w = get_tensor(p + ".mlp.0.weight"); + b.mlp_in_b = get_tensor(p + ".mlp.0.bias"); + b.mlp_out_w = get_tensor(p + ".mlp.2.weight"); + b.mlp_out_b = get_tensor(p + ".mlp.2.bias"); + c.s3tok_blocks.push_back(b); + } + c.s3tok_down_w = get_tensor("a.s3tok.quantizer._codebook.project_down.weight"); + c.s3tok_down_b = get_tensor("a.s3tok.quantizer._codebook.project_down.bias"); + + // read on the host: embedding tables, source module + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { + const std::string name = t->name; + if (name.rfind("a.gen.code.", 0) == 0 || name.rfind("a.gen.t3.", 0) == 0 || + name.rfind("a.gen.hift.m_source.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + } + } + + // voice. holds the talker rows, voice..ref_* the flow decoder reference + const int64_t n_embd_talker = model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; + for (int64_t i = 0; i < gguf_get_n_tensors(ctx_gguf.get()); ++i) { + const std::string name = gguf_get_tensor_name(ctx_gguf.get(), i); + if (name.rfind("voice.", 0) != 0) { + continue; + } + const ggml_tensor * t = ggml_get_tensor(ctx_meta.get(), name.c_str()); + std::string voice = name.substr(6); + std::vector ids; // the converter stores 1D tensors as F32, ids are exact + void * dst = nullptr; + bool ok = false; + auto take = [&](const char * suffix) { + const size_t n = strlen(suffix); + if (voice.size() > n && voice.compare(voice.size() - n, n, suffix) == 0) { + voice.resize(voice.size() - n); + return true; + } + return false; + }; + if (take(".ref_tokens")) { + ids.resize(ggml_nelements(t)); + dst = ids.data(); + ok = t->type == GGML_TYPE_F32; + } else if (take(".ref_feat")) { + auto & v = model.gen_voices[voice].ref.feat; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && t->ne[0] == 80; + } else if (take(".ref_spk")) { + auto & v = model.gen_voices[voice].ref.spk; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && t->ne[0] == 192; + } else { + auto & v = model.gen_voices[voice].rows; + v.resize(ggml_nelements(t)); + dst = v.data(); + ok = t->type == GGML_TYPE_F32 && ggml_n_dims(t) <= 2 && t->ne[0] == n_embd_talker; + } + if (!ok) { + throw std::runtime_error(string_format("%s: invalid voice tensor %s\n", __func__, name.c_str())); + } + fin.seekg(tensor_offset.at(name), std::ios::beg); + if (!fin.read(reinterpret_cast(dst), ggml_nbytes(t))) { + throw std::runtime_error(string_format("%s: failed to read %s\n", __func__, name.c_str())); + } + if (!ids.empty()) { + model.gen_voices[voice].ref.tokens.assign(ids.begin(), ids.end()); + } + } + for (const auto & [voice, v] : model.gen_voices) { + if (v.rows.empty() || v.ref.tokens.empty() || v.ref.feat.size() != 2 * 80 * v.ref.tokens.size() || v.ref.spk.empty()) { + throw std::runtime_error(string_format("%s: incomplete voice %s\n", __func__, voice.c_str())); + } + } + } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + auto & c = model.cbx; + auto has = [&](const std::string & name) { + return gguf_find_tensor(ctx_gguf.get(), name.c_str()) >= 0; + }; + auto load_bn = [&](const std::string & p, clip_chatterbox::bn & n) { + n.mean = get_tensor(p + ".running_mean"); + n.var = get_tensor(p + ".running_var"); + n.w = get_tensor(p + ".weight", false); + n.b = get_tensor(p + ".bias", false); + }; + auto load_res2d = [&](const std::string & p, clip_chatterbox::spk_res2d & r) { + r.conv1_w = get_tensor(p + ".conv1.weight"); + load_bn(p + ".bn1", r.bn1); + r.conv2_w = get_tensor(p + ".conv2.weight"); + load_bn(p + ".bn2", r.bn2); + r.shortcut_w = get_tensor(p + ".shortcut.0.weight", false); + if (r.shortcut_w) { + load_bn(p + ".shortcut.1", r.shortcut_bn); + } + }; + auto load_cam_block = [&](const std::string & bp, const std::string & tp, clip_chatterbox::spk_cam_block & blk) { + for (int li = 1; has(bp + ".tdnnd" + std::to_string(li) + ".linear1.weight"); li++) { + const std::string p = bp + ".tdnnd" + std::to_string(li); + clip_chatterbox::spk_cam_layer l; + load_bn(p + ".nonlinear1.batchnorm", l.nl1_bn); + l.linear1_w = get_tensor(p + ".linear1.weight"); + load_bn(p + ".nonlinear2.batchnorm", l.nl2_bn); + l.local_w = get_tensor(p + ".cam_layer.linear_local.weight"); + l.ctx1_w = get_tensor(p + ".cam_layer.linear1.weight"); + l.ctx1_b = get_tensor(p + ".cam_layer.linear1.bias"); + l.ctx2_w = get_tensor(p + ".cam_layer.linear2.weight"); + l.ctx2_b = get_tensor(p + ".cam_layer.linear2.bias"); + blk.layers.push_back(l); + } + load_bn(tp + ".nonlinear.batchnorm", blk.transit_bn); + blk.transit_w = get_tensor(tp + ".linear.weight"); + }; + + c.spk_conv1_w = get_tensor("a.spk.head.conv1.weight"); + load_bn("a.spk.head.bn1", c.spk_bn1); + load_res2d("a.spk.head.layer1.0", c.spk_layer1_0); + load_res2d("a.spk.head.layer1.1", c.spk_layer1_1); + load_res2d("a.spk.head.layer2.0", c.spk_layer2_0); + load_res2d("a.spk.head.layer2.1", c.spk_layer2_1); + c.spk_conv2_w = get_tensor("a.spk.head.conv2.weight"); + load_bn("a.spk.head.bn2", c.spk_bn2); + c.spk_tdnn_w = get_tensor("a.spk.xvector.tdnn.linear.weight"); + load_bn("a.spk.xvector.tdnn.nonlinear.batchnorm", c.spk_tdnn_bn); + load_cam_block("a.spk.xvector.block1", "a.spk.xvector.transit1", c.spk_block1); + load_cam_block("a.spk.xvector.block2", "a.spk.xvector.transit2", c.spk_block2); + load_cam_block("a.spk.xvector.block3", "a.spk.xvector.transit3", c.spk_block3); + load_bn("a.spk.xvector.out_nonlinear.batchnorm", c.spk_out_bn); + c.spk_dense_w = get_tensor("a.spk.xvector.dense.linear.weight"); + load_bn("a.spk.xvector.dense.nonlinear.batchnorm", c.spk_dense_bn); + + // read on the host: voice encoder lstm, conditioning encoder + for (ggml_tensor * t = ggml_get_first_tensor(ctx_meta.get()); t; t = ggml_get_next_tensor(ctx_meta.get(), t)) { + const std::string name = t->name; + if (name.rfind("a.ve.", 0) == 0 || name.rfind("a.cenc.", 0) == 0) { + model.cbx_tensors[name] = get_tensor(name); + } + } + } break; case PROJECTOR_TYPE_SOPRANO: { auto & v = model.vocos; @@ -4741,8 +5095,19 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_gen_audio->flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED; } loader.load_tensors(*ctx_gen_audio); + if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_CHATTERBOX) { + // the classic cfm solver unrolls up to 10 steps x 2 cfg estimator evaluations in one graph + ctx_gen_audio->max_nodes = 65536; + ctx_gen_audio->sched.reset( + ggml_backend_sched_new(ctx_gen_audio->backend_ptrs.data(), ctx_gen_audio->backend_buft.data(), + ctx_gen_audio->backend_ptrs.size(), ctx_gen_audio->max_nodes, false, true)); + if (ctx_params.cb_eval != nullptr) { + ggml_backend_sched_set_eval_callback(ctx_gen_audio->sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data); + } + } // TODO: fix warmup - ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead()); + ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + + ggml_graph_overhead_custom(ctx_gen_audio->max_nodes, false)); } } catch (const std::exception & e) { @@ -5148,6 +5513,11 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) { // pooling gives one speaker embedding, whatever the clip length is n_patches = 1; } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // statistics pooling gives one speaker embedding, whatever the clip length is + n_patches = 1; + } break; case PROJECTOR_TYPE_QWEN3TTS_GEN: case PROJECTOR_TYPE_LFM3A_GEN: { @@ -5276,7 +5646,7 @@ void clip_gen_state_reset(clip_gen_state * state) { } } -const std::map> & clip_gen_get_voices(const clip_ctx * ctx) { +const std::map & clip_gen_get_voices(const clip_ctx * ctx) { return ctx->model.gen_voices; } @@ -5354,6 +5724,32 @@ void clip_enc_state_free(clip_enc_state * state) { delete state; } +size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max) { + const auto & tensors = ctx->model.cbx_tensors; + auto it = tensors.find(name); + if (it == tensors.end() || !it->second) { + return 0; + } + ggml_tensor * t = it->second; + const size_t n = (size_t) ggml_nelements(t); + if (!out) { + return n; + } + if (n_max < n) { + return 0; + } + if (t->type == GGML_TYPE_F32) { + ggml_backend_tensor_get(t, out, 0, n * sizeof(float)); + } else if (t->type == GGML_TYPE_F16) { + std::vector tmp(n); + ggml_backend_tensor_get(t, tmp.data(), 0, n * sizeof(ggml_fp16_t)); + for (size_t i = 0; i < n; i++) out[i] = ggml_fp16_to_fp32(tmp[i]); + } else { + return 0; + } + return n; +} + bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { const clip_image_f32_batch & imgs = *params->imgs; int n_batch_cur = imgs.entries.size(); @@ -5369,7 +5765,10 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { clip_model_loader::warmup(*ctx, *params->imgs); } - if (params->gen_process != CLIP_GEN_PROCESS_GEN_UNKNOWN && params->gen_state == nullptr) { + const bool needs_gen_state = params->gen_process == CLIP_GEN_PROCESS_GEN_CODE || + params->gen_process == CLIP_GEN_PROCESS_GEN_WAV || + params->gen_process == CLIP_GEN_PROCESS_TTS; + if (needs_gen_state && params->gen_state == nullptr) { LOG_ERR("%s: audio gen requires a gen state\n", __func__); return false; } @@ -5543,8 +5942,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } set_input_f32("inp_raw", inp_raw); - } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV && ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { - // audio input. GEN_WAV is not here: it takes codes or feats, set in the switch below. + } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV && + params->gen_process != CLIP_GEN_PROCESS_TTS && + params->gen_process != CLIP_GEN_PROCESS_TTS_VOCODE && + ggml_graph_get_tensor(gf, "inp_raw") != nullptr) { + // audio input. GEN_WAV and the chatterbox TTS stages are not here: they take codes or feats, set in the switch below. // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding) GGML_ASSERT(imgs.entries.size() == 1); @@ -6337,6 +6739,84 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_attn_mask(qwen2_mask); } } break; + case PROJECTOR_TYPE_CHATTERBOX: + { + if (params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { + set_input_f32("inp_mel", *params->mel_in); + set_input_f32("inp_sstft", *params->sstft_in); + break; + } + if (params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) { + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int T2 = (T1 - 1) / 2 + 1; + std::vector pos(T2); + for (int i = 0; i < T2; i++) { + pos[(size_t) i] = i; + } + set_input_i32("inp_pos", pos); + break; + } + if (params->gen_process != CLIP_GEN_PROCESS_TTS) { + break; + } + const int n_gen = (int) params->codes->size(); + GGML_ASSERT(params->ref); + const int n_prompt = (int) params->ref->tokens.size(); + std::vector tokens = params->ref->tokens; + tokens.resize((size_t) n_prompt + n_gen); + memcpy(tokens.data() + n_prompt, params->codes->data(), (size_t) n_gen * sizeof(int32_t)); + const int64_t n_codes = model.cbx.input_embedding_w->ne[1]; + for (int32_t t : tokens) { + if (t < 0 || t >= n_codes) { + LOG_ERR("%s: speech token out of range (%d, codebook size %lld)\n", __func__, t, (long long) n_codes); + return false; + } + } + const int T1 = n_prompt + n_gen; + const int T2 = 2 * T1; + set_input_i32("inp_tokens", tokens); + + // mel-rate reference conditioning of the voice + set_input_f32("inp_prompt_feat", params->ref->feat); + set_input_f32("inp_xvec", params->ref->spk); + + // espnet relative positional encoding, entry k holds the sinusoid of relative position (T-1) - k + auto fill_pos = [&](const char * name, int T) { + const int d = 512; + std::vector pos((size_t) (2 * T - 1) * d); + for (int k = 0; k < 2 * T - 1; k++) { + const double rel = (double) (T - 1 - k); + for (int i = 0; i < d / 2; i++) { + const double div = exp(-(double) (2 * i) * log(10000.0) / d); + pos[(size_t) k * d + 2 * i ] = (float) sin(rel * div); + pos[(size_t) k * d + 2 * i + 1] = (float) cos(rel * div); + } + } + set_input_f32(name, pos); + }; + fill_pos("inp_pos1", T1); + fill_pos("inp_pos2", T2); + + // noise and the time embeddings of the solver span points, matcha layout (sines then cosines, scale 1000) + std::vector noise((size_t) 80 * T2); + std::normal_distribution nd(0.0f, 1.0f); + for (auto & f : noise) f = nd(params->gen_state->rng); + set_input_f32("inp_noise", noise); + + const bool meanflow = model.cbx.time_embed_mixer_w != nullptr; + const int n_steps = meanflow ? 2 : 10; + std::vector temb((size_t) 320 * (n_steps + 1)); + for (int s = 0; s <= n_steps; s++) { + const double u = (double) s / n_steps; + const double t = meanflow ? u : 1.0 - cos(u * M_PI / 2.0); + for (int i = 0; i < 160; i++) { + const double div = exp(-(double) i * log(10000.0) / 159.0); + temb[(size_t) s * 320 + i ] = (float) sin(1000.0 * t * div); + temb[(size_t) s * 320 + i + 160] = (float) cos(1000.0 * t * div); + } + } + set_input_f32("inp_temb", temb); + } break; case PROJECTOR_TYPE_GEMMA3: case PROJECTOR_TYPE_GEMMA3NV: case PROJECTOR_TYPE_IDEFICS3: @@ -6363,6 +6843,16 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { { // do nothing } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + // the ceil-mode seg pooling divides by the full segment length, rescale the last partial segment + const int T1 = (imgs.entries[0].nx() - 1) / 2 + 1; + const int S = (T1 + 99) / 100; + std::vector segfix((size_t) S, 1.0f); + const int last = T1 - (S - 1) * 100; + segfix[(size_t) S - 1] = 100.0f / (float) last; + set_input_f32("inp_segfix", segfix); + } break; case PROJECTOR_TYPE_LFM3A_GEN: { const int64_t n_cb = model.gen_code_embd_w->ne[2]; @@ -6966,6 +7456,210 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { // for audio gen models // + if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TOKENIZE) { + ggml_tensor * fsq = ggml_graph_get_tensor(gf, "out_fsq"); + GGML_ASSERT(fsq != nullptr && params->out_codes); + const int n_tok = (int) fsq->ne[1]; + std::vector h((size_t) 8 * n_tok); + ggml_backend_tensor_get(fsq, h.data(), 0, ggml_nbytes(fsq)); + + // fsq round to base 3: h in (-1, 1) maps to digits {0, 1, 2} + params->out_codes->resize(n_tok); + for (int t = 0; t < n_tok; t++) { + int32_t code = 0; + for (int i = 7; i >= 0; i--) { + const int32_t d = (int32_t) roundf(h[(size_t) t * 8 + i] * 0.9990000128746033f) + 1; + code = code * 3 + d; + } + (*params->out_codes)[(size_t) t] = code; + } + + // speech table rows of the codes, the multilingual variant adds its learned positions as its perceiver expects + if (params->out_code_embd) { + const auto & tensors = ctx->model.cbx_tensors; + auto tab_it = tensors.find("a.gen.code.out_embd.weight"); + GGML_ASSERT(tab_it != tensors.end()); + ggml_tensor * tab = tab_it->second; + const int n_e = (int) tab->ne[0]; + auto pos_it = tensors.find("a.gen.t3.speech_pos_emb"); + ggml_tensor * pos = pos_it == tensors.end() ? nullptr : pos_it->second; + if ((tab->type != GGML_TYPE_F16 && tab->type != GGML_TYPE_F32) || + (pos && (pos->type != GGML_TYPE_F32 || pos->ne[0] != n_e || pos->ne[1] < n_tok))) { + LOG_ERR("%s: unexpected speech table or positions\n", __func__); + return false; + } + + params->out_code_embd->resize((size_t) n_tok * n_e); + std::vector h16(n_e); + std::vector pr(n_e); + for (int t = 0; t < n_tok; t++) { + float * dst = params->out_code_embd->data() + (size_t) t * n_e; + if ((*params->out_codes)[(size_t) t] >= tab->ne[1]) { + LOG_ERR("%s: speech code beyond the speech table\n", __func__); + return false; + } + const size_t r = (size_t) (*params->out_codes)[(size_t) t] * n_e; + if (tab->type == GGML_TYPE_F16) { + ggml_backend_tensor_get(tab, h16.data(), r * sizeof(ggml_fp16_t), (size_t) n_e * sizeof(ggml_fp16_t)); + for (int j = 0; j < n_e; j++) { + dst[j] = ggml_fp16_to_fp32(h16[j]); + } + } else { + ggml_backend_tensor_get(tab, dst, r * sizeof(float), (size_t) n_e * sizeof(float)); + } + if (pos) { + ggml_backend_tensor_get(pos, pr.data(), (size_t) t * n_e * sizeof(float), (size_t) n_e * sizeof(float)); + for (int j = 0; j < n_e; j++) { + dst[j] += pr[j]; + } + } + } + } + return true; + } + + if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { + ggml_tensor * sp = ggml_graph_get_tensor(gf, "out_spec"); + GGML_ASSERT(sp != nullptr && params->out_spec); + params->out_spec->resize(ggml_nelements(sp)); + ggml_backend_tensor_get(sp, params->out_spec->data(), 0, ggml_nbytes(sp)); + return true; + } + + if (ctx->proj_type() == PROJECTOR_TYPE_CHATTERBOX && params->gen_process == CLIP_GEN_PROCESS_TTS) { + ggml_tensor * mel = ggml_graph_get_tensor(gf, "out_mel"); + GGML_ASSERT(mel != nullptr); + std::vector mel_data(ggml_nelements(mel)); + ggml_backend_tensor_get(mel, mel_data.data(), 0, ggml_nbytes(mel)); + + // hift: f0 -> harmonic source -> source stft on the host, then the vocoder graph and the istft + ggml_tensor * f0_t = ggml_graph_get_tensor(gf, "out_f0"); + GGML_ASSERT(f0_t != nullptr); + const int n_mel_out = (int) ggml_nelements(f0_t); + std::vector f0(n_mel_out); + ggml_backend_tensor_get(f0_t, f0.data(), 0, ggml_nbytes(f0_t)); + + // SineGen: f0 upsampled x480, 9 harmonics with cumulative phase, uv gating and noise, merged by l_linear + tanh + const int ups_total = 480; + const double sr = 24000.0; + const int64_t n_wav = (int64_t) n_mel_out * ups_total; + std::vector lw(9); + float lb = 0.0f; + if (clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.weight", lw.data(), lw.size()) != 9 || + clip_cbx_read_tensor(ctx, "a.gen.hift.m_source.l_linear.bias", &lb, 1) != 1) { + LOG_ERR("%s: unexpected hift source module\n", __func__); + return false; + } + + std::mt19937 & srng = params->gen_state->rng; + std::uniform_real_distribution ud(-M_PI, M_PI); + std::normal_distribution snd(0.0f, 1.0f); + double phase[9]; + for (int h = 0; h < 9; h++) { + phase[h] = h == 0 ? 0.0 : ud(srng); + } + std::vector src((size_t) n_wav); + double cum[9] = {0.0}; + for (int64_t t = 0; t < n_wav; t++) { + const float f = f0[(size_t) (t / ups_total)]; + const float uv = f > 10.0f ? 1.0f : 0.0f; // nsf_voiced_threshold + const float namp = uv * 0.003f + (1.0f - uv) * 0.1f / 3.0f; + float merged = lb; + for (int h = 0; h < 9; h++) { + cum[h] += (double) f * (h + 1) / sr; + cum[h] -= floor(cum[h]); + float sine = 0.1f * (float) sin(2.0 * M_PI * cum[h] + phase[h]); + sine = sine * uv + namp * snd(srng); + merged += lw[(size_t) h] * sine; + } + src[(size_t) t] = tanhf(merged); + } + + // stft of the source: n_fft 16, hop 4, hann window, centered + const int n_fft = 16, hop = 4, n_bins = 9; + std::vector win(n_fft); + for (int i = 0; i < n_fft; i++) win[(size_t) i] = 0.5f - 0.5f * cosf(2.0f * (float) M_PI * i / n_fft); + const int n_stft = (int) (n_wav / hop) + 1; + std::vector sstft((size_t) n_stft * 18); + for (int fr = 0; fr < n_stft; fr++) { + const int64_t c0 = (int64_t) fr * hop - n_fft / 2; // centered, reflect padded + for (int k = 0; k < n_bins; k++) { + double re = 0.0, im = 0.0; + for (int i = 0; i < n_fft; i++) { + int64_t idx = c0 + i; + if (idx < 0) idx = -idx; + if (idx >= n_wav) idx = 2 * (n_wav - 1) - idx; + const double v = (double) src[(size_t) idx] * win[(size_t) i]; + const double a = 2.0 * M_PI * k * i / n_fft; + re += v * cos(a); + im -= v * sin(a); + } + sstft[(size_t) fr * 18 + k ] = (float) re; + sstft[(size_t) fr * 18 + 9 + k] = (float) im; + } + } + + // vocoder graph on mel + source stft + std::vector spec; + { + clip_encode_params vp = *params; + vp.gen_process = CLIP_GEN_PROCESS_TTS_VOCODE; + vp.mel_in = &mel_data; + vp.sstft_in = &sstft; + vp.vocode_n_mel = n_mel_out; + vp.vocode_n_stft = n_stft; + vp.out_audio = nullptr; + vp.out_spec = &spec; + if (!clip_encode(ctx, &vp)) { + LOG_ERR("%s: vocoder stage failed\n", __func__); + return false; + } + } + + // istft: mag = clipped exp, phase = sin, hann overlap-add + const int n_frames_out = (int) (spec.size() / 18); + const int64_t n_out = (int64_t) (n_frames_out - 1) * hop; + std::vector acc((size_t) n_out + n_fft, 0.0); + std::vector wsum((size_t) n_out + n_fft, 0.0); + for (int fr = 0; fr < n_frames_out; fr++) { + double frame[16]; + for (int i = 0; i < n_fft; i++) { + double v = 0.0; + for (int k = 0; k < n_bins; k++) { + const double mag = fmin(exp((double) spec[(size_t) fr * 18 + k]), 1e2); + const double ph = sin((double) spec[(size_t) fr * 18 + 9 + k]); + const double re = mag * cos(ph), im = mag * sin(ph); + const double a = 2.0 * M_PI * k * i / n_fft; + const double w = (k == 0 || k == n_fft / 2) ? 1.0 : 2.0; + v += w * (re * cos(a) - im * sin(a)); + } + frame[i] = v / n_fft; + } + const int64_t o = (int64_t) fr * hop; + for (int i = 0; i < n_fft; i++) { + acc [(size_t) (o + i)] += frame[i] * win[(size_t) i]; + wsum[(size_t) (o + i)] += (double) win[(size_t) i] * win[(size_t) i]; + } + } + GGML_ASSERT(params->out_audio); + auto & out_audio = *params->out_audio; + out_audio.resize((size_t) std::min(n_out - n_fft / 2, n_wav)); + for (size_t i = 0; i < out_audio.size(); i++) { + const size_t j = i + n_fft / 2; // drop the centering pad + const double v = wsum[j] > 1e-11 ? acc[j] / wsum[j] : 0.0; + out_audio[(size_t) i] = (float) fmax(-0.99, fmin(0.99, v)); + } + + // trim_fade of the reference: silence 20 ms then fade in 20 ms, hides the artifact of the prompt boundary + const size_t n_trim = 24000 / 50; + for (size_t i = 0; i < 2 * n_trim && i < out_audio.size(); i++) { + const double g = i < n_trim ? 0.0 + : (cos(M_PI * (1.0 - (double) (i - n_trim) / n_trim)) + 1.0) / 2.0; + out_audio[(size_t) i] *= (float) g; + } + return true; + } + // optional outputs: a pipeline yields codes or feats, and not all have an eos head if (params->out_codes != nullptr) { ggml_tensor * codes = ggml_graph_get_tensor(gf, "out_codes"); @@ -7235,6 +7929,12 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) { return ctx->model.spk_proj_w->ne[1]; case PROJECTOR_TYPE_POCKETTTS_GEN: return ctx->model.gen_input_lin_w->ne[1]; + case PROJECTOR_TYPE_CHATTERBOX: + // talker width, carried by the speech table + return ctx->model.cbx_tensors.at("a.gen.code.out_embd.weight")->ne[0]; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + // talker conditioning rows, the speaker projection carries their width + return ctx->model.cbx_tensors.at("a.cenc.spkr_enc.weight")->ne[1]; case PROJECTOR_TYPE_SOPRANO: return ctx->model.hparams.projection_dim; case PROJECTOR_TYPE_PARAKEET: diff --git a/tools/mtmd/clip.h b/tools/mtmd/clip.h index b2c9e07597c7..6c8dcf31cc05 100644 --- a/tools/mtmd/clip.h +++ b/tools/mtmd/clip.h @@ -96,7 +96,18 @@ enum clip_gen_process_type { CLIP_GEN_PROCESS_GEN_UNKNOWN, CLIP_GEN_PROCESS_GEN_CODE, // h_state to codes CLIP_GEN_PROCESS_GEN_WAV, // codes to raw PCM audio + CLIP_GEN_PROCESS_TTS, // full utterance of codes to raw PCM audio + CLIP_GEN_PROCESS_TTS_VOCODE, // internal: mel + source stft to istft input + CLIP_GEN_PROCESS_TOKENIZE, // raw PCM audio to semantic speech tokens }; + +// gen-side voice reference (chatterbox): speech tokens, mel-rate features and the speaker vector +struct clip_gen_ref { + std::vector tokens; + std::vector feat; + std::vector spk; +}; + struct clip_encode_params { int n_threads = 1; const clip_image_f32_batch * imgs = nullptr; @@ -114,10 +125,20 @@ struct clip_encode_params { float temp = 0.0f; // sampling temperature, noise scale for flow-matching decoders bool * out_is_eos = nullptr; + // TOKENIZE: speech embedding rows of the produced codes + std::vector * out_code_embd = nullptr; + // GEN_WAV const std::vector * codes = nullptr; // this frame's 16 RVQ codes const std::vector * feats = nullptr; // continuous counterpart of codes std::vector * out_audio = nullptr; // decoded PCM samples, F32 + const clip_gen_ref * ref = nullptr; // TTS reference conditioning of the voice + // TTS_VOCODE internal stage inputs + const std::vector * mel_in = nullptr; + const std::vector * sstft_in = nullptr; + int vocode_n_mel = 0; + int vocode_n_stft = 0; + std::vector * out_spec = nullptr; // audio gen: state of one sequence, clip_encode() updates it in place struct clip_gen_state * gen_state = nullptr; @@ -139,7 +160,16 @@ struct clip_gen_state * clip_gen_state_init(struct clip_ctx * ctx, uint32_t seed void clip_gen_state_free(struct clip_gen_state * state); void clip_gen_state_reset(struct clip_gen_state * state); -const std::map> & clip_gen_get_voices(const struct clip_ctx * ctx); +// read a chatterbox tensor by name as F32, returns the element count (0 if not found), out may be null to query the size +size_t clip_cbx_read_tensor(struct clip_ctx * ctx, const char * name, float * out, size_t n_max); + +// a predefined voice: backbone rows, plus the gen-side reference when the gen stack needs one +struct clip_gen_voice { + std::vector rows; + clip_gen_ref ref; +}; + +const std::map & clip_gen_get_voices(const struct clip_ctx * ctx); bool clip_is_llava(const struct clip_ctx * ctx); // note for contributor: this clip_is_(model) pattern is deprecated diff --git a/tools/mtmd/models/chatterbox-gen.cpp b/tools/mtmd/models/chatterbox-gen.cpp new file mode 100644 index 000000000000..d4c4f77bf740 --- /dev/null +++ b/tools/mtmd/models/chatterbox-gen.cpp @@ -0,0 +1,542 @@ +#define _USE_MATH_DEFINES // for M_PI +#include "models.h" + +// Chatterbox generation graphs: flow encoder, cfm estimator, s3 tokenizer and hift vocoder + +// x [C, T]: y = W x + b with torch Linear weights stored as [in, out] +ggml_tensor * clip_graph_chatterbox_base::cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const { + ggml_tensor * y = ggml_mul_mat(ctx0, w, x); + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// conv1d over time with asymmetric zero padding: x [C, T] -> [OC, T_out], kernel [K, IC, OC] +ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int stride, int pad_l, int pad_r) const { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T, C] + if (pad_l > 0) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_l, xt->ne[1]); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, z, xt, 0); + } + if (pad_r > 0) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, pad_r, xt->ne[1]); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, xt, z, 0); + } + ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, stride, 0, 1); // [T_out, OC] + y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [OC, T_out] + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// x [C, T] -> symmetric-padded dilated conv -> [OC, T] +ggml_tensor * clip_graph_chatterbox_base::cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int pad, int dil) const { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + ggml_tensor * y = ggml_conv_1d(ctx0, k, xt, 1, pad, dil); + y = ggml_cont(ctx0, ggml_transpose(ctx0, y)); + if (b) { + y = ggml_add(ctx0, y, b); + } + return y; +} + +// Transformer-XL relative shift: bd [2T-1, T, H] -> [T, T, H] with out[j, i, h] = bd[(T-1) - i + j, i, h] +// same buffer walk as the espnet rel_shift: pad one column, drop the first row, keep the first T columns +static ggml_tensor * cbx_rel_shift(ggml_context * ctx0, ggml_tensor * bd, int T) { + const int H = (int) bd->ne[2]; + ggml_tensor * z = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, T, H); + z = ggml_scale(ctx0, z, 0.0f); + ggml_tensor * p = ggml_cont(ctx0, ggml_concat(ctx0, z, bd, 0)); // [2T, T, H] + p = ggml_reshape_3d(ctx0, p, T, 2 * T, H); // [T, 2T, H] + p = ggml_view_3d(ctx0, p, T, 2 * T - 1, H, p->nb[1], p->nb[2], p->nb[1]); // drop first row + p = ggml_cont(ctx0, p); + p = ggml_reshape_3d(ctx0, p, 2 * T - 1, T, H); // [2T-1, T, H] + p = ggml_view_3d(ctx0, p, T, T, H, p->nb[1], p->nb[2], 0); // first T columns + return ggml_cont(ctx0, p); +} + +// espnet rel-pos self attention block, pre-norm, x [512, T], pos [512, 2T-1] +ggml_tensor * clip_graph_chatterbox::enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, + ggml_tensor * pos, int T) { + const int n_head = 8; + const int d_head = 64; + const float scale = 1.0f / sqrtf((float) d_head); + + ggml_tensor * res = x; + ggml_tensor * cur = build_norm(x, l.norm_mha_w, l.norm_mha_b, NORM_TYPE_NORMAL, 1e-5f, -1); + + ggml_tensor * q = cbx_linear(l.attn_q_w, l.attn_q_b, cur); + ggml_tensor * k = cbx_linear(l.attn_k_w, l.attn_k_b, cur); + ggml_tensor * v = cbx_linear(l.attn_v_w, l.attn_v_b, cur); + ggml_tensor * pe = cbx_linear(l.attn_pos_w, nullptr, pos); // [512, 2T-1] + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, T); + k = ggml_reshape_3d(ctx0, k, d_head, n_head, T); + v = ggml_reshape_3d(ctx0, v, d_head, n_head, T); + pe = ggml_reshape_3d(ctx0, pe, d_head, n_head, 2 * T - 1); + + ggml_tensor * u = l.attn_pos_bias_u; // [64, 8] + ggml_tensor * w = l.attn_pos_bias_v; + + ggml_tensor * qu = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, u, d_head, n_head, 1)); + ggml_tensor * qv = ggml_add(ctx0, q, ggml_reshape_3d(ctx0, w, d_head, n_head, 1)); + + // per head: [64, T] tensors, scores [T(k), T(q)] + qu = ggml_cont(ctx0, ggml_permute(ctx0, qu, 0, 2, 1, 3)); // [64, T, 8] + qv = ggml_cont(ctx0, ggml_permute(ctx0, qv, 0, 2, 1, 3)); + k = ggml_cont(ctx0, ggml_permute(ctx0, k, 0, 2, 1, 3)); + v = ggml_cont(ctx0, ggml_permute(ctx0, v, 0, 2, 1, 3)); + pe = ggml_cont(ctx0, ggml_permute(ctx0, pe, 0, 2, 1, 3)); // [64, 2T-1, 8] + + ggml_tensor * ac = ggml_mul_mat(ctx0, k, qu); // [T(k), T(q), 8] + ggml_tensor * bd = ggml_mul_mat(ctx0, pe, qv); // [2T-1, T(q), 8] + bd = cbx_rel_shift(ctx0, bd, T); // [T(k), T(q), 8] + + ggml_tensor * scores = ggml_scale(ctx0, ggml_add(ctx0, ac, bd), scale); + ggml_tensor * probs = ggml_soft_max(ctx0, scores); + + ggml_tensor * o = ggml_mul_mat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, v)), probs); // [64, T, 8] + o = ggml_cont(ctx0, ggml_permute(ctx0, o, 0, 2, 1, 3)); // [64, 8, T] + o = ggml_reshape_2d(ctx0, o, n_head * d_head, T); + o = cbx_linear(l.attn_out_w, l.attn_out_b, o); + x = ggml_add(ctx0, res, o); + + res = x; + cur = build_norm(x, l.norm_ff_w, l.norm_ff_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(l.ffn_1_w, l.ffn_1_b, cur); + cur = ggml_silu(ctx0, cur); // swish + cur = cbx_linear(l.ffn_2_w, l.ffn_2_b, cur); + x = ggml_add(ctx0, res, cur); + return x; +} + +// mish = x * tanh(softplus(x)) +static ggml_tensor * cbx_mish(ggml_context * ctx0, ggml_tensor * x) { + return ggml_mul(ctx0, x, ggml_tanh(ctx0, ggml_softplus(ctx0, x))); +} + +// causal block: conv k3 left-padded, layer norm over channels, mish; x [C, T] +ggml_tensor * clip_graph_chatterbox::causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x) { + x = cbx_conv1d(b.conv_w, b.conv_b, x, 1, (int) b.conv_w->ne[0] - 1, 0); + x = build_norm(x, b.norm_w, b.norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + return cbx_mish(ctx0, x); +} + +// resnet block with time conditioning; x [C, T], temb [1024] +ggml_tensor * clip_graph_chatterbox::resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb) { + ggml_tensor * h = causal_block(r.block1, x); + ggml_tensor * tproj = cbx_linear(r.mlp_w, r.mlp_b, cbx_mish(ctx0, temb)); + h = ggml_add(ctx0, h, tproj); // broadcast [256, 1] over T + h = causal_block(r.block2, h); + ggml_tensor * res = cbx_conv1d(r.res_conv_w, r.res_conv_b, x, 1, 0, 0); + return ggml_add(ctx0, h, res); +} + +// diffusers-style transformer block, full attention; x [256, T] +ggml_tensor * clip_graph_chatterbox::tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x) { + const int n_head = 8; + const int d_head = 64; + const int T = (int) x->ne[1]; + + ggml_tensor * cur = build_norm(x, b.norm1_w, b.norm1_b, NORM_TYPE_NORMAL, 1e-5f, -1); + ggml_tensor * q = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_q_w, cur), d_head, n_head, T); + ggml_tensor * k = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_k_w, cur), d_head, n_head, T); + ggml_tensor * v = ggml_reshape_3d(ctx0, ggml_mul_mat(ctx0, b.attn_v_w, cur), d_head, n_head, T); + x = ggml_add(ctx0, x, build_attn(b.attn_out_w, b.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1)); + + cur = build_norm(x, b.norm3_w, b.norm3_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(b.ff_in_w, b.ff_in_b, cur); + cur = ggml_gelu_erf(ctx0, cur); + cur = cbx_linear(b.ff_out_w, b.ff_out_b, cur); + return ggml_add(ctx0, x, cur); +} + +// one estimator evaluation; x_noise, mu, cond [80, T], spks [80], temb [1024] +ggml_tensor * clip_graph_chatterbox::estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, + ggml_tensor * cond, ggml_tensor * temb, int T) { + const auto & c = model.cbx; + + // channels live on ne0, time on ne1: pack along ne0 + ggml_tensor * x = ggml_concat(ctx0, x_noise, mu, 0); // [160, T] + ggml_tensor * spks_b = ggml_repeat(ctx0, ggml_reshape_2d(ctx0, spks, 80, 1), ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T)); + x = ggml_concat(ctx0, x, spks_b, 0); // [240, T] + x = ggml_concat(ctx0, x, cond, 0); // [320, T] + x = ggml_cont(ctx0, x); + + // down + ggml_tensor * skip; + x = resnet(c.est_down.res, x, temb); + for (const auto & b : c.est_down.tfm) { + x = tfm_block(b, x); + } + skip = x; + x = cbx_conv1d(c.est_down.conv_w, c.est_down.conv_b, x, 1, (int) c.est_down.conv_w->ne[0] - 1, 0); + + // mid + for (const auto & m : c.est_mid) { + x = resnet(m.res, x, temb); + for (const auto & b : m.tfm) { + x = tfm_block(b, x); + } + } + + // up with skip + x = ggml_concat(ctx0, x, skip, 0); // [512, T] + x = resnet(c.est_up.res, x, temb); + for (const auto & b : c.est_up.tfm) { + x = tfm_block(b, x); + } + x = cbx_conv1d(c.est_up.conv_w, c.est_up.conv_b, x, 1, (int) c.est_up.conv_w->ne[0] - 1, 0); + + x = causal_block(c.est_final_block, x); + x = cbx_conv1d(c.est_final_proj_w, c.est_final_proj_b, x, 1, 0, 0); // [80, T] + return x; +} + +// s3 tokenizer encoder: log-mel [T, 128] -> fsq code [8, T / 4], rounded to base 3 tokens on the host +ggml_cgraph * clip_graph_chatterbox::build_s3tok(int T) { + const auto & c = model.cbx; + const int n_head = 20; + const int d_head = 64; + const int T1 = (T - 1) / 2 + 1; + const int T2 = (T1 - 1) / 2 + 1; + + ggml_tensor * inp = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, T, 128); + ggml_set_name(inp, "inp_raw"); + ggml_set_input(inp); + + ggml_tensor * pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T2); + ggml_set_name(pos, "inp_pos"); + ggml_set_input(pos); + + ggml_tensor * x = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); // [128, T] + x = cbx_conv1d(c.s3tok_conv1_w, c.s3tok_conv1_b, x, 2, 1, 1); + x = ggml_gelu_erf(ctx0, x); + x = cbx_conv1d(c.s3tok_conv2_w, c.s3tok_conv2_b, x, 2, 1, 1); + x = ggml_gelu_erf(ctx0, x); // [1280, T2] + + for (const auto & blk : c.s3tok_blocks) { + ggml_tensor * res = x; + ggml_tensor * cur = build_norm(x, blk.attn_ln_w, blk.attn_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); + ggml_tensor * q = cbx_linear(blk.attn_q_w, blk.attn_q_b, cur); + ggml_tensor * k = ggml_mul_mat(ctx0, blk.attn_k_w, cur); + ggml_tensor * v = cbx_linear(blk.attn_v_w, blk.attn_v_b, cur); + + // fsmn memory: depthwise conv over time on v plus residual, added to the attention output + ggml_tensor * fsm = ggml_cont(ctx0, ggml_transpose(ctx0, v)); // [T2, 1280] + { + ggml_tensor * w = blk.fsmn_w; + ggml_tensor * m = ggml_conv_1d_dw(ctx0, w, fsm, 1, ((int) w->ne[0] - 1) / 2, 1); + fsm = ggml_add(ctx0, ggml_reshape_2d(ctx0, m, fsm->ne[0], fsm->ne[1]), fsm); + } + fsm = ggml_cont(ctx0, ggml_transpose(ctx0, fsm)); // [1280, T2] + + q = ggml_reshape_3d(ctx0, q, d_head, n_head, T2); + k = ggml_reshape_3d(ctx0, k, d_head, n_head, T2); + v = ggml_reshape_3d(ctx0, v, d_head, n_head, T2); + q = ggml_rope_ext(ctx0, q, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + k = ggml_rope_ext(ctx0, k, pos, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f); + ggml_tensor * o = build_attn(blk.attn_out_w, blk.attn_out_b, q, k, v, nullptr, 1.0f / sqrtf((float) d_head), -1); + x = ggml_add(ctx0, res, ggml_add(ctx0, o, fsm)); + + res = x; + cur = build_norm(x, blk.mlp_ln_w, blk.mlp_ln_b, NORM_TYPE_NORMAL, 1e-5f, -1); + cur = cbx_linear(blk.mlp_in_w, blk.mlp_in_b, cur); + cur = ggml_gelu_erf(ctx0, cur); + cur = cbx_linear(blk.mlp_out_w, blk.mlp_out_b, cur); + x = ggml_add(ctx0, res, cur); + } + + x = cbx_linear(c.s3tok_down_w, c.s3tok_down_b, x); // [8, T2] + x = ggml_tanh(ctx0, x); + + ggml_set_name(x, "out_fsq"); + ggml_set_output(x); + ggml_build_forward_expand(gf, x); + return gf; +} + +ggml_cgraph * clip_graph_chatterbox::build() { + if (gen_process == CLIP_GEN_PROCESS_TTS_VOCODE) { + return build_vocoder(vocode_n_mel, vocode_n_stft); + } + if (gen_process == CLIP_GEN_PROCESS_TOKENIZE) { + return build_s3tok(img.nx()); + } + if (gen_process != CLIP_GEN_PROCESS_TTS) { + // load-time buffer sizing path + ggml_tensor * inp = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 1); + ggml_set_name(inp, "inp_stub"); + ggml_set_input(inp); + ggml_tensor * cur = ggml_dup(ctx0, inp); + ggml_set_name(cur, "out_stub"); + ggml_set_output(cur); + ggml_build_forward_expand(gf, cur); + return gf; + } + + const auto & c = model.cbx; + + const int T1 = n_tokens; // token-rate length (prompt + generated) + const int T2 = 2 * n_tokens; // mel-rate length after the x2 upsample + + ggml_tensor * inp_tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, T1); + ggml_set_name(inp_tokens, "inp_tokens"); + ggml_set_input(inp_tokens); + + ggml_tensor * pos1 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T1 - 1); + ggml_set_name(pos1, "inp_pos1"); + ggml_set_input(pos1); + + ggml_tensor * pos2 = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, 2 * T2 - 1); + ggml_set_name(pos2, "inp_pos2"); + ggml_set_input(pos2); + + // token embedding + ggml_tensor * x = ggml_get_rows(ctx0, c.input_embedding_w, inp_tokens); // [512, T1] + + // embed: linear + layer norm, then the espnet xscale + x = cbx_linear(c.embed_linear_w, c.embed_linear_b, x); + x = build_norm(x, c.embed_norm_w, c.embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + x = ggml_scale(ctx0, x, sqrtf(512.0f)); + cb(x, "fenc_embd", -1); + + // pre-lookahead: conv k=4 right-padded 3, leaky 0.01, conv k=3 left-padded 2, residual + { + ggml_tensor * res = x; + ggml_tensor * cur = cbx_conv1d(c.pre_conv1_w, c.pre_conv1_b, x, 1, 0, 3); + cur = ggml_leaky_relu(ctx0, cur, 0.01f, false); + cur = cbx_conv1d(c.pre_conv2_w, c.pre_conv2_b, cur, 1, 2, 0); + x = ggml_add(ctx0, res, cur); + cb(x, "fenc_pre_lookahead", -1); + } + + for (size_t i = 0; i < c.enc.size(); i++) { + x = enc_layer(c.enc[i], x, pos1, T1); + cb(x, "fenc_enc", (int) i); + } + + // upsample x2: nearest repeat, left pad 4, conv k=5 + { + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512] + xt = ggml_interpolate(ctx0, xt, 2 * T1, 512, 1, 1, GGML_SCALE_MODE_NEAREST); + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 4, 512); + z = ggml_scale(ctx0, z, 0.0f); + xt = ggml_concat(ctx0, z, xt, 0); + ggml_tensor * y = ggml_conv_1d(ctx0, c.up_conv_w, xt, 1, 0, 1); + x = ggml_cont(ctx0, ggml_transpose(ctx0, y)); // [512, T2] + x = ggml_add(ctx0, x, c.up_conv_b); + cb(x, "fenc_upsample", -1); + } + + // up embed: linear + layer norm + xscale + x = cbx_linear(c.up_embed_linear_w, c.up_embed_linear_b, x); + x = build_norm(x, c.up_embed_norm_w, c.up_embed_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + x = ggml_scale(ctx0, x, sqrtf(512.0f)); + + for (size_t i = 0; i < c.up_enc.size(); i++) { + x = enc_layer(c.up_enc[i], x, pos2, T2); + cb(x, "fenc_up_enc", (int) i); + } + + x = build_norm(x, c.after_norm_w, c.after_norm_b, NORM_TYPE_NORMAL, 1e-5f, -1); + + // encoder projection to the mel channel count + ggml_tensor * mu = cbx_linear(c.encoder_proj_w, c.encoder_proj_b, x); // [80, T2] + cb(mu, "flow_mu", -1); + + // cfm solver unrolled in the graph: meanflow takes 2 euler steps without cfg, its time embeds mix t and r + // the classic solver takes 10 euler steps on the cosine schedule with cfg 0.7 + const bool meanflow = c.time_embed_mixer_w != nullptr; + const int n_steps = meanflow ? 2 : 10; + + // span points, same schedule as the host side sinusoid fill in clip.cpp + float span[11]; + for (int i = 0; i <= n_steps; i++) { + const float u = (float) i / n_steps; + span[i] = meanflow ? u : 1.0f - cosf(u * (float) M_PI / 2.0f); + } + + ggml_tensor * noise = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2); + ggml_set_name(noise, "inp_noise"); + ggml_set_input(noise); + // sinusoidal time embeddings, one row per span point + ggml_tensor * temb_sin = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 320, n_steps + 1); + ggml_set_name(temb_sin, "inp_temb"); + ggml_set_input(temb_sin); + + auto time_mlp = [&](ggml_tensor * e) { + e = cbx_linear(c.time_mlp_1_w, c.time_mlp_1_b, e); + e = ggml_silu(ctx0, e); + e = cbx_linear(c.time_mlp_2_w, c.time_mlp_2_b, e); + return e; + }; + auto span_emb = [&](int i) { + return ggml_view_2d(ctx0, temb_sin, 320, 1, temb_sin->nb[1], (size_t) i * temb_sin->nb[1]); + }; + auto step_temb = [&](int i) { + if (!meanflow) { + return time_mlp(span_emb(i)); + } + ggml_tensor * e = ggml_concat(ctx0, time_mlp(span_emb(i)), time_mlp(span_emb(i + 1)), 0); // [2048, 1] + return ggml_mul_mat(ctx0, c.time_embed_mixer_w, e); // [1024, 1] + }; + + // mel-rate conditions of the voice: prompt features then zeros, and the speaker vector + ggml_tensor * pf = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_prompt_mel); + ggml_set_name(pf, "inp_prompt_feat"); + ggml_set_input(pf); + ggml_tensor * zc = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, T2 - n_prompt_mel); + zc = ggml_scale(ctx0, zc, 0.0f); + ggml_tensor * cond = ggml_concat(ctx0, pf, zc, 1); // [80, T2] + // raw 192-dim campplus x-vector, normalized then through the flow speaker affine + ggml_tensor * xvec = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, 192); + ggml_set_name(xvec, "inp_xvec"); + ggml_set_input(xvec); + ggml_tensor * n2 = ggml_sqrt(ctx0, ggml_sum(ctx0, ggml_mul(ctx0, xvec, xvec))); + ggml_tensor * unit = ggml_div(ctx0, xvec, n2); + ggml_tensor * spks = cbx_linear(c.spk_affine_w, c.spk_affine_b, + ggml_reshape_2d(ctx0, unit, 192, 1)); + spks = ggml_cont(ctx0, ggml_reshape_1d(ctx0, spks, 80)); + + const float cfg = 0.7f; + ggml_tensor * mu_zero = meanflow ? nullptr : ggml_scale(ctx0, mu, 0.0f); + ggml_tensor * cond_zero = meanflow ? nullptr : ggml_scale(ctx0, cond, 0.0f); + ggml_tensor * spks_zero = meanflow ? nullptr : ggml_scale(ctx0, spks, 0.0f); + + ggml_tensor * mx = noise; + for (int i = 0; i < n_steps; i++) { + ggml_tensor * temb = step_temb(i); + ggml_tensor * d = estimator(mx, mu, spks, cond, temb, T2); + if (!meanflow) { + ggml_tensor * du = estimator(mx, mu_zero, spks_zero, cond_zero, temb, T2); + d = ggml_add(ctx0, ggml_scale(ctx0, d, 1.0f + cfg), ggml_scale(ctx0, du, -cfg)); + } + mx = ggml_add(ctx0, mx, ggml_scale(ctx0, d, span[i + 1] - span[i])); + cb(mx, "cfm_step", i); + } + ggml_tensor * mel = mx; + + // trim the prompt frames at mel rate + mel = ggml_view_2d(ctx0, mel, 80, T2 - n_prompt_mel, mel->nb[1], (size_t) n_prompt_mel * mel->nb[1]); + mel = ggml_cont(ctx0, mel); + ggml_set_name(mel, "out_mel"); + ggml_set_output(mel); + ggml_build_forward_expand(gf, mel); + + // f0 predictor on the trimmed mel: 5x (conv k3 same-pad + elu), abs(linear) + { + ggml_tensor * fx = mel; + for (const auto & fc : c.f0_condnet) { + fx = cbx_conv1d(fc.w, fc.b, fx, 1, 1, 1); + fx = ggml_elu(ctx0, fx); + } + fx = cbx_linear(c.f0_classifier_w, c.f0_classifier_b, fx); + fx = ggml_abs(ctx0, fx); // [1, T] + ggml_set_name(fx, "out_f0"); + ggml_set_output(fx); + ggml_build_forward_expand(gf, fx); + } + return gf; +} + +// snake activation with per-channel alpha: x + sin^2(alpha x) / alpha +static ggml_tensor * cbx_snake(ggml_context * ctx0, ggml_tensor * x, ggml_tensor * alpha) { + ggml_tensor * sx = ggml_sin(ctx0, ggml_mul(ctx0, x, alpha)); + sx = ggml_mul(ctx0, sx, sx); + sx = ggml_div(ctx0, sx, alpha); + return ggml_add(ctx0, x, sx); +} + +// hifigan-snake resblock; kernels with dilations 1/3/5 on convs1, 1 on convs2 +ggml_tensor * clip_graph_chatterbox::hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x) { + static const int dil[3] = {1, 3, 5}; + for (size_t j = 0; j < r.units.size(); j++) { + const auto & u = r.units[j]; + const int d = dil[j % 3]; + const int p1 = (int) (u.conv1_w->ne[0] - 1) / 2 * d; + const int p2 = (int) (u.conv2_w->ne[0] - 1) / 2; + ggml_tensor * xt = cbx_snake(ctx0, x, u.act1_alpha); + xt = cbx_conv1d_dil(u.conv1_w, u.conv1_b, xt, p1, d); + xt = cbx_snake(ctx0, xt, u.act2_alpha); + xt = cbx_conv1d_dil(u.conv2_w, u.conv2_b, xt, p2, 1); + x = ggml_add(ctx0, x, xt); + } + return x; +} + +// mel [80, T] + source stft [18, T_stft] -> conv_post output [18, T_stft2] +ggml_cgraph * clip_graph_chatterbox::build_vocoder(int n_mel, int n_stft) { + const auto & c = model.cbx; + + ggml_tensor * mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 80, n_mel); + ggml_set_name(mel, "inp_mel"); + ggml_set_input(mel); + ggml_tensor * sstft = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 18, n_stft); + ggml_set_name(sstft, "inp_sstft"); + ggml_set_input(sstft); + + ggml_tensor * x = cbx_conv1d_dil(c.hift_pre_w, c.hift_pre_b, mel, 3, 1); + + for (size_t i = 0; i < c.hift_ups.size(); i++) { + const auto & up = c.hift_ups[i]; + ggml_tensor * uk = up.up_w; + const int K = (int) uk->ne[0]; + const int S = K / 2; + const int P = (K - S) / 2; + + x = ggml_leaky_relu(ctx0, x, 0.1f, false); + // conv transpose then trim the torch padding P on both sides + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + xt = ggml_conv_transpose_1d(ctx0, uk, xt, S, 0, 1); + xt = ggml_cont(ctx0, ggml_view_2d(ctx0, xt, xt->ne[0] - 2 * P, xt->ne[1], xt->nb[1], (size_t) P * ggml_element_size(xt))); + x = ggml_cont(ctx0, ggml_transpose(ctx0, xt)); + x = ggml_add(ctx0, x, up.up_b); + + const bool is_last = i + 1 == c.hift_ups.size(); + if (is_last) { + ggml_tensor * xr = ggml_cont(ctx0, ggml_transpose(ctx0, x)); + xr = ggml_pad_reflect_1d(ctx0, xr, 1, 0); + x = ggml_cont(ctx0, ggml_transpose(ctx0, xr)); + } + + // source injection: strided conv on the source stft, one resblock + ggml_tensor * sk = up.source_down_w; + const int SK = (int) sk->ne[0]; + const int SS = SK > 1 ? SK / 2 : 1; + const int SP = SK > 1 ? SS / 2 : 0; + ggml_tensor * si; + { + ggml_tensor * st = ggml_cont(ctx0, ggml_transpose(ctx0, sstft)); + st = ggml_conv_1d(ctx0, sk, st, SS, SP, 1); + si = ggml_cont(ctx0, ggml_transpose(ctx0, st)); + si = ggml_add(ctx0, si, up.source_down_b); + } + si = hift_resblock(up.source_res, si); + // align lengths: the reflection pad on the last stage adds one step + if ((int) si->ne[1] != (int) x->ne[1]) { + const int n = (int) std::min(si->ne[1], x->ne[1]); + si = ggml_cont(ctx0, ggml_view_2d(ctx0, si, si->ne[0], n, si->nb[1], 0)); + x = ggml_cont(ctx0, ggml_view_2d(ctx0, x, x->ne[0], n, x->nb[1], 0)); + } + x = ggml_add(ctx0, x, si); + + ggml_tensor * acc = hift_resblock(up.res_0, x); + acc = ggml_add(ctx0, acc, hift_resblock(up.res_1, x)); + acc = ggml_add(ctx0, acc, hift_resblock(up.res_2, x)); + x = ggml_scale(ctx0, acc, 1.0f / 3.0f); + } + + x = ggml_leaky_relu(ctx0, x, 0.01f, false); + x = cbx_conv1d_dil(c.hift_post_w, c.hift_post_b, x, 3, 1); + ggml_set_name(x, "out_spec"); + ggml_set_output(x); + ggml_build_forward_expand(gf, x); + return gf; +} diff --git a/tools/mtmd/models/chatterbox-spkenc.cpp b/tools/mtmd/models/chatterbox-spkenc.cpp new file mode 100644 index 000000000000..afd9f0c29042 --- /dev/null +++ b/tools/mtmd/models/chatterbox-spkenc.cpp @@ -0,0 +1,156 @@ +#include "models.h" + +// Chatterbox speaker encoder: CAMPPlus x-vector on kaldi fbank features, see s3gen/xvector.py + +static constexpr float bn_eps = 1e-5f; + +// per-channel batchnorm on x [C, T], w/b are null on the affine=False variant +ggml_tensor * clip_graph_chatterbox_spkenc::bn1d(const clip_chatterbox::bn & n, ggml_tensor * x) { + ggml_tensor * sd = ggml_sqrt(ctx0, ggml_scale_bias(ctx0, n.var, 1.0f, bn_eps)); + if (!n.w) { + return ggml_div(ctx0, ggml_sub(ctx0, x, n.mean), sd); + } + ggml_tensor * a = ggml_div(ctx0, n.w, sd); + ggml_tensor * shift = ggml_sub(ctx0, n.b, ggml_mul(ctx0, n.mean, a)); + return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift); +} + +// batchnorm on a conv2d activation [W=T, H=F, C, 1], stats on ne2 +static ggml_tensor * cbx_bn2d(ggml_context * ctx0, const clip_chatterbox::bn & n, ggml_tensor * x) { + const int C = (int) x->ne[2]; + ggml_tensor * mean = ggml_reshape_4d(ctx0, n.mean, 1, 1, C, 1); + ggml_tensor * var = ggml_reshape_4d(ctx0, n.var, 1, 1, C, 1); + ggml_tensor * w = ggml_reshape_4d(ctx0, n.w, 1, 1, C, 1); + ggml_tensor * b = ggml_reshape_4d(ctx0, n.b, 1, 1, C, 1); + ggml_tensor * a = ggml_div(ctx0, w, ggml_sqrt(ctx0, ggml_scale_bias(ctx0, var, 1.0f, bn_eps))); + ggml_tensor * shift = ggml_sub(ctx0, b, ggml_mul(ctx0, mean, a)); + return ggml_add(ctx0, ggml_mul(ctx0, x, a), shift); +} + +ggml_tensor * clip_graph_chatterbox_spkenc::bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x) { + return ggml_relu(ctx0, cbx_bn2d(ctx0, n, x)); +} + +// fcm residual 2d block, stride on the frequency axis only +ggml_tensor * clip_graph_chatterbox_spkenc::res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x, + int stride) { + ggml_tensor * cur = ggml_conv_2d(ctx0, r.conv1_w, x, 1, stride, 1, 1, 1, 1); + cur = bn2d_relu(r.bn1, cur); + cur = ggml_conv_2d(ctx0, r.conv2_w, cur, 1, 1, 1, 1, 1, 1); + // bn2 without the relu, applied before the residual add + cur = cbx_bn2d(ctx0, r.bn2, cur); + ggml_tensor * res = x; + if (r.shortcut_w) { + res = ggml_conv_2d(ctx0, r.shortcut_w, x, 1, stride, 0, 0, 1, 1); + res = cbx_bn2d(ctx0, r.shortcut_bn, res); + } + return ggml_relu(ctx0, ggml_add(ctx0, cur, res)); +} + +// cam dense tdnn layer: bottleneck then context-gated conv; x [C_in, T] -> [growth, T] +ggml_tensor * clip_graph_chatterbox_spkenc::cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x, + int dil, ggml_tensor * segfix) { + ggml_tensor * h = ggml_relu(ctx0, bn1d(l.nl1_bn, x)); + h = cbx_conv1d(l.linear1_w, nullptr, h, 1, 0, 0); + h = ggml_relu(ctx0, bn1d(l.nl2_bn, h)); + + ggml_tensor * k = l.local_w; + const int pad = ((int) k->ne[0] - 1) / 2 * dil; + ggml_tensor * y = cbx_conv1d_dil(k, nullptr, h, pad, dil); + + // context: global mean plus ceil-mode segment means of length 100 + const int T = (int) h->ne[1]; + const int C = (int) h->ne[0]; + const int S = (T + 99) / 100; + ggml_tensor * ht = ggml_cont(ctx0, ggml_transpose(ctx0, h)); // [T, C] + ggml_tensor * gmean = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mean(ctx0, ht))); // [C, 1] + ggml_tensor * seg; + { + ggml_tensor * padded = ht; + if (S * 100 != T) { + ggml_tensor * z = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, S * 100 - T, C); + z = ggml_scale(ctx0, z, 0.0f); + padded = ggml_concat(ctx0, ht, z, 0); + } + ggml_tensor * pooled = ggml_pool_1d(ctx0, padded, GGML_OP_POOL_AVG, 100, 100, 0); // [S, C] + pooled = ggml_mul(ctx0, pooled, segfix); + ggml_tensor * exp = ggml_interpolate(ctx0, pooled, S * 100, C, 1, 1, GGML_SCALE_MODE_NEAREST); + exp = ggml_cont(ctx0, ggml_view_2d(ctx0, exp, T, C, exp->nb[1], 0)); + seg = ggml_cont(ctx0, ggml_transpose(ctx0, exp)); // [C, T] + } + ggml_tensor * context = ggml_add(ctx0, seg, gmean); + context = cbx_conv1d(l.ctx1_w, l.ctx1_b, context, 1, 0, 0); + context = ggml_relu(ctx0, context); + context = cbx_conv1d(l.ctx2_w, l.ctx2_b, context, 1, 0, 0); + ggml_tensor * m = ggml_sigmoid(ctx0, context); + return ggml_mul(ctx0, y, m); +} + +ggml_cgraph * clip_graph_chatterbox_spkenc::build() { + const auto & c = model.cbx; + + const int T = img.nx(); + const int T1 = (T - 1) / 2 + 1; + const int S = (T1 + 99) / 100; + ggml_tensor * segfix = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, S); + ggml_set_name(segfix, "inp_segfix"); + ggml_set_input(segfix); + + // fbank features [T, 80] from the preprocessor + ggml_tensor * inp = build_inp_raw(1); + + // fcm 2d front: [W=T, H=F=80, C=1] -> [T, 10, 32] -> [320, T] + ggml_tensor * x = ggml_reshape_4d(ctx0, inp, T, 80, 1, 1); + x = ggml_conv_2d(ctx0, c.spk_conv1_w, x, 1, 1, 1, 1, 1, 1); + x = bn2d_relu(c.spk_bn1, x); + x = res2d(c.spk_layer1_0, x, 2); + x = res2d(c.spk_layer1_1, x, 1); + x = res2d(c.spk_layer2_0, x, 2); + x = res2d(c.spk_layer2_1, x, 1); + x = ggml_conv_2d(ctx0, c.spk_conv2_w, x, 1, 2, 1, 1, 1, 1); + x = bn2d_relu(c.spk_bn2, x); + x = ggml_reshape_2d(ctx0, x, T, 320); + x = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [320, T] + cb(x, "spk_fcm", -1); + + // tdnn k5 stride 2 over time, then the three cam dense blocks + x = cbx_conv1d(c.spk_tdnn_w, nullptr, x, 2, 2, 2); // [128, T1] + x = ggml_relu(ctx0, bn1d(c.spk_tdnn_bn, x)); + cb(x, "spk_tdnn", -1); + + static const int block_dil[3] = {1, 2, 2}; + const clip_chatterbox::spk_cam_block * blocks[3] = { &c.spk_block1, &c.spk_block2, &c.spk_block3 }; + for (int bi = 1; bi <= 3; bi++) { + const auto & blk = *blocks[bi - 1]; + for (const auto & l : blk.layers) { + ggml_tensor * out = cam_layer(l, x, block_dil[bi - 1], segfix); + x = ggml_concat(ctx0, x, out, 0); + } + x = ggml_relu(ctx0, bn1d(blk.transit_bn, x)); + x = cbx_conv1d(blk.transit_w, nullptr, x, 1, 0, 0); + cb(x, "spk_block", bi); + } + x = ggml_relu(ctx0, bn1d(c.spk_out_bn, x)); // [512, T1] + + // statistics pooling: mean and unbiased std over time -> [1024, 1] + ggml_tensor * xt = ggml_cont(ctx0, ggml_transpose(ctx0, x)); // [T1, 512] + ggml_tensor * mean = ggml_mean(ctx0, xt); // [1, 512] + ggml_tensor * m2 = ggml_mean(ctx0, ggml_mul(ctx0, xt, xt)); + ggml_tensor * var = ggml_sub(ctx0, m2, ggml_mul(ctx0, mean, mean)); + var = ggml_scale(ctx0, var, (float) T1 / (float) (T1 - 1)); + ggml_tensor * sd = ggml_sqrt(ctx0, ggml_relu(ctx0, var)); + ggml_tensor * stats = ggml_concat(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, mean)), + ggml_cont(ctx0, ggml_transpose(ctx0, sd)), 0); // [1024, 1] + cb(stats, "spk_stats_pool", -1); + + // the x-vector stays raw, the gen graph normalizes it + ggml_tensor * dw = ggml_reshape_2d(ctx0, c.spk_dense_w, 1024, 192); + ggml_tensor * emb = ggml_mul_mat(ctx0, dw, stats); // [192, 1] + emb = bn1d(c.spk_dense_bn, emb); + emb = ggml_cont(ctx0, ggml_reshape_1d(ctx0, emb, 192)); + cb(emb, "spk_embd", -1); + ggml_set_name(emb, "out_xvec"); + ggml_set_output(emb); + ggml_build_forward_expand(gf, emb); + return gf; +} diff --git a/tools/mtmd/models/models.h b/tools/mtmd/models/models.h index dc816fa87c34..c7520468f4e9 100644 --- a/tools/mtmd/models/models.h +++ b/tools/mtmd/models/models.h @@ -206,6 +206,53 @@ struct clip_graph_conformer : clip_graph { ggml_cgraph * build() override; }; +// builders shared by the chatterbox gen and spkenc graphs, defined in chatterbox-gen.cpp +struct clip_graph_chatterbox_base : clip_graph { + using clip_graph::clip_graph; + + protected: + ggml_tensor * cbx_linear(ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) const; + ggml_tensor * cbx_conv1d(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int stride, int pad_l, int pad_r) const; + ggml_tensor * cbx_conv1d_dil(ggml_tensor * k, ggml_tensor * b, ggml_tensor * x, + int pad, int dil) const; +}; + +struct clip_graph_chatterbox_spkenc : clip_graph_chatterbox_base { + clip_graph_chatterbox_spkenc(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_chatterbox_base(ctx, img) {} + ggml_cgraph * build() override; + + private: + ggml_tensor * bn1d(const clip_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * bn2d_relu(const clip_chatterbox::bn & n, ggml_tensor * x); + ggml_tensor * res2d(const clip_chatterbox::spk_res2d & r, ggml_tensor * x, int stride); + ggml_tensor * cam_layer(const clip_chatterbox::spk_cam_layer & l, ggml_tensor * x, int dil, ggml_tensor * segfix); +}; + +struct clip_graph_chatterbox : clip_graph_chatterbox_base { + clip_gen_process_type gen_process = CLIP_GEN_PROCESS_GEN_CODE; + int n_tokens = 0; + int n_prompt_mel = 0; + int vocode_n_mel = 0; + int vocode_n_stft = 0; + clip_graph_chatterbox(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, int n_tokens, + int n_prompt_mel, int vocode_n_mel, int vocode_n_stft) + : clip_graph_chatterbox_base(ctx, img), gen_process(gen_process), n_tokens(n_tokens), n_prompt_mel(n_prompt_mel), + vocode_n_mel(vocode_n_mel), vocode_n_stft(vocode_n_stft) {} + ggml_cgraph * build() override; + + private: + ggml_tensor * enc_layer(const clip_chatterbox::enc_layer & l, ggml_tensor * x, ggml_tensor * pos, int T); + ggml_tensor * causal_block(const clip_chatterbox::causal_block & b, ggml_tensor * x); + ggml_tensor * resnet(const clip_chatterbox::resnet & r, ggml_tensor * x, ggml_tensor * temb); + ggml_tensor * tfm_block(const clip_chatterbox::tfm_block & b, ggml_tensor * x); + ggml_tensor * estimator(ggml_tensor * x_noise, ggml_tensor * mu, ggml_tensor * spks, + ggml_tensor * cond, ggml_tensor * temb, int T); + ggml_tensor * hift_resblock(const clip_chatterbox::hift_res & r, ggml_tensor * x); + ggml_cgraph * build_s3tok(int T); + ggml_cgraph * build_vocoder(int n_mel, int n_stft); +}; + struct clip_graph_lfm3a : clip_graph { clip_graph_lfm3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {} ggml_cgraph * build() override; diff --git a/tools/mtmd/mtmd-audio.cpp b/tools/mtmd/mtmd-audio.cpp index 6e1ee23242d3..74d26552655f 100644 --- a/tools/mtmd/mtmd-audio.cpp +++ b/tools/mtmd/mtmd-audio.cpp @@ -1,6 +1,7 @@ #include "mtmd-audio.h" #define _USE_MATH_DEFINES // for M_PI +#include #include #include #include @@ -970,6 +971,480 @@ bool mtmd_audio_preprocessor_qwen3tts_spk::preprocess(const float * return true; } +// windowed frame of a centered, reflect padded signal: dst[i] = x[start + i] * win[i], zeros past the window +static void mtmd_audio_reflect_frame(const float * x, int n, int start, const std::vector & win, std::vector & dst) { + std::fill(dst.begin(), dst.end(), 0.0f); + for (int i = 0; i < (int) win.size(); i++) { + int idx = start + i; + if (idx < 0) { + idx = -idx; + } + if (idx >= n) { + idx = 2 * n - 2 - idx; + } + dst[(size_t) i] = x[idx] * win[(size_t) i]; + } +} + +// mel energies of the frame in fft_in: power (or magnitude) spectrum through cache.filters +static void mtmd_audio_mel_energies(const mtmd_audio_cache & cache, std::vector & fft_in, std::vector & fft_out, + bool magnitude, std::vector & mel) { + const int n_fft = (int) cache.filters.n_fft; + const int n_bins = n_fft / 2 + 1; + fft(cache, fft_in.data(), n_fft, fft_out.data()); + for (int k = 0; k < n_bins; k++) { + const float p = fft_out[2 * k + 0] * fft_out[2 * k + 0] + fft_out[2 * k + 1] * fft_out[2 * k + 1]; + fft_out[k] = magnitude ? sqrtf(p + 1e-9f) : p; + } + mel.assign(cache.filters.n_mel, 0.0); + for (int64_t m = 0; m < cache.filters.n_mel; m++) { + const float * w = cache.filters.data.data() + (size_t) m * n_bins; + for (int k = 0; k < n_bins; k++) { + mel[(size_t) m] += w[k] * fft_out[k]; + } + } +} + +// whisper style log-mel of the chatterbox s3 tokenizer (s3tokenizer.py), mel major +static bool mtmd_audio_s3tok_log_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 400; + const int hop = 160; + const int n_mel = (int) cache.filters.n_mel; + const int n = (int) n_samples; + + n_frames = n / hop; + if (n_frames <= 0) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + std::vector mel((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - n_fft / 2, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + mel[(size_t) m * n_frames + fr] = log10(std::max(e[(size_t) m], 1e-10)); + } + } + + const double mx = *std::max_element(mel.begin(), mel.end()); + out.resize(mel.size()); + for (size_t i = 0; i < mel.size(); i++) { + out[i] = (float) ((std::max(mel[i], mx - 8.0) + 4.0) / 4.0); + } + return true; +} + +// 16 kHz -> 24 kHz upsampler: hann windowed sinc cut just under the source nyquist +static void mtmd_audio_upsample_3_2(const float * samples, size_t n_samples, std::vector & out) { + const int W = 16; // sinc half width in source samples + const double fc = 0.495; // cutoff, normalized to the source rate + + const size_t n_out = n_samples * 3 / 2; + out.assign(n_out, 0.0f); + for (size_t n = 0; n < n_out; n++) { + const double t = (double) (2 * n) / 3.0; + const int k0 = (int) floor(t) - W + 1; + double acc = 0.0; + for (int k = k0; k < k0 + 2 * W; k++) { + if (k < 0 || k >= (int) n_samples) { + continue; + } + const double x = t - k; + const double s = x == 0.0 ? 1.0 : sin(2.0 * M_PI * fc * x) / (M_PI * x); + acc += samples[k] * s * 0.5 * (1.0 + cos(M_PI * x / (W + 1))); + } + out[n] = (float) acc; + } +} + +// matcha style log-mel of the s3gen prompt features (s3gen/utils/mel.py), frame major +static bool mtmd_audio_matcha_log_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 1920; + const int hop = 480; + const int pad = (n_fft - hop) / 2; + const int n_mel = 80; + const int n = (int) n_samples; + + n_frames = 1 + (n + 2 * pad - n_fft) / hop; + if (n <= pad || n_frames <= 0) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + out.resize((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - pad, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, true, e); + for (int m = 0; m < n_mel; m++) { + out[(size_t) fr * n_mel + m] = logf(std::max((float) e[(size_t) m], 1e-5f)); + } + } + return true; +} + +// librosa.effects.trim: non-silent where the rms window is within top_db of the peak +static void mtmd_audio_trim_silence(const float * samples, size_t n_samples, float top_db, + size_t & start, size_t & end) { + const int win = 2048; + const int hop = 512; + const int n = (int) n_samples; + + const int n_fr = 1 + n / hop; + std::vector rms((size_t) n_fr); + double mx = 0.0; + for (int fr = 0; fr < n_fr; fr++) { + double acc = 0.0; + for (int i = 0; i < win; i++) { + const int idx = fr * hop - win / 2 + i; + if (idx >= 0 && idx < n) { + acc += (double) samples[idx] * samples[idx]; + } + } + rms[(size_t) fr] = sqrt(acc / win); + mx = std::max(mx, rms[(size_t) fr]); + } + + const double thr = mx * pow(10.0, -top_db / 20.0); + int first = -1, last = -1; + for (int fr = 0; fr < n_fr; fr++) { + if (rms[(size_t) fr] > thr) { + if (first < 0) { + first = fr; + } + last = fr; + } + } + if (first < 0) { + start = end = 0; + return; + } + start = (size_t) first * hop; + end = std::min((size_t) (last + 1) * hop, n_samples); +} + +// power mel of the voice encoder front-end, frame major, no log +static bool mtmd_audio_ve_mel(const mtmd_audio_cache & cache, const float * samples, size_t n_samples, + std::vector & out, int & n_frames) { + const int n_fft = 400; + const int hop = 160; + const int n_mel = 40; + const int n = (int) n_samples; + + n_frames = 1 + n / hop; + if (n < 2) { + return false; + } + + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + out.resize((size_t) n_mel * n_frames); + for (int fr = 0; fr < n_frames; fr++) { + mtmd_audio_reflect_frame(samples, n, fr * hop - n_fft / 2, cache.hann_window, fft_in); + mtmd_audio_mel_energies(cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + out[(size_t) fr * n_mel + m] = (float) e[(size_t) m]; + } + } + return true; +} + +// ITU-R BS.1770 integrated loudness of a mono signal, matching pyloudnorm +static float mtmd_audio_lufs(const float * samples, size_t n_samples, int sample_rate) { + std::vector y(samples, samples + n_samples); + + auto biquad = [&](double b0, double b1, double b2, double a1, double a2) { + double x1 = 0.0, x2 = 0.0, y1 = 0.0, y2 = 0.0; + for (double & v : y) { + const double x0 = v; + v = b0 * x0 + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2; + x2 = x1; x1 = x0; + y2 = y1; y1 = v; + } + }; + + // stage 1: high shelf, f0 1681.9744509555319 Hz, +3.99984385397 dB, Q 0.7071752369554196 + { + const double A = pow(10.0, 3.99984385397 / 40.0); + const double w0 = 2.0 * M_PI * 1681.9744509555319 / sample_rate; + const double alpha = sin(w0) / (2.0 * 0.7071752369554196); + const double c = cos(w0); + const double sq = 2.0 * sqrt(A) * alpha; + const double b0 = A * ((A + 1.0) + (A - 1.0) * c + sq); + const double b1 = -2.0 * A * ((A - 1.0) + (A + 1.0) * c); + const double b2 = A * ((A + 1.0) + (A - 1.0) * c - sq); + const double a0 = (A + 1.0) - (A - 1.0) * c + sq; + const double a1 = 2.0 * ((A - 1.0) - (A + 1.0) * c); + const double a2 = (A + 1.0) - (A - 1.0) * c - sq; + biquad(b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0); + } + // stage 2: high pass, f0 38.13547087602444 Hz, Q 0.5003270373238773 + { + const double w0 = 2.0 * M_PI * 38.13547087602444 / sample_rate; + const double alpha = sin(w0) / (2.0 * 0.5003270373238773); + const double c = cos(w0); + const double b0 = (1.0 + c) / 2.0; + const double b1 = -(1.0 + c); + const double b2 = (1.0 + c) / 2.0; + const double a0 = 1.0 + alpha; + const double a1 = -2.0 * c; + const double a2 = 1.0 - alpha; + biquad(b0 / a0, b1 / a0, b2 / a0, a1 / a0, a2 / a0); + } + + const int win = (int) (0.4 * sample_rate); + const int hop = win / 4; + if ((int) n_samples < win) { + return -HUGE_VALF; + } + const int n_blocks = 1 + ((int) n_samples - win) / hop; + std::vector z((size_t) n_blocks); + for (int b = 0; b < n_blocks; b++) { + double acc = 0.0; + for (int i = 0; i < win; i++) { + acc += y[(size_t) b * hop + i] * y[(size_t) b * hop + i]; + } + z[(size_t) b] = acc / win; + } + + auto gated_mean = [&](double thr_lufs) { + double acc = 0.0; + int n = 0; + for (const double v : z) { + if (-0.691 + 10.0 * log10(std::max(v, 1e-30)) > thr_lufs) { + acc += v; + n++; + } + } + return n > 0 ? acc / n : 0.0; + }; + + const double z_abs = gated_mean(-70.0); + if (z_abs <= 0.0) { + return -HUGE_VALF; + } + const double thr_rel = -0.691 + 10.0 * log10(z_abs) - 10.0; + const double z_rel = gated_mean(thr_rel); + if (z_rel <= 0.0) { + return -HUGE_VALF; + } + return (float) (-0.691 + 10.0 * log10(z_rel)); +} + +// +// mtmd_audio_preprocessor_chatterbox_ref +// + +void mtmd_audio_preprocessor_chatterbox_ref::initialize() { + // kaldi fbank: povey window, kaldi mel banks over the 512-point spectrum, the nyquist bin left out + { + const int frame_len = 400; + const int n_fft = 512; + const int n_bins = n_fft / 2 + 1; + const int n_mel = hparams.n_mel_bins; + const double sr = (double) hparams.audio_sample_rate; + + fbank_cache.fill_sin_cos_table(n_fft); + fbank_cache.hann_window.resize(frame_len); + for (int i = 0; i < frame_len; i++) { + fbank_cache.hann_window[(size_t) i] = (float) pow(0.5 - 0.5 * cos(2.0 * M_PI * i / (frame_len - 1)), 0.85); + } + + auto mel = [](double f) { return 1127.0 * log(1.0 + f / 700.0); }; + const double mel_lo = mel(20.0); + const double mel_hi = mel(sr / 2.0); + const double delta = (mel_hi - mel_lo) / (n_mel + 1); + const double bin_hz = sr / n_fft; + + auto & filters = fbank_cache.filters; + filters.n_mel = n_mel; + filters.n_fft = n_fft; + filters.data.assign((size_t) n_mel * n_bins, 0.0f); + for (int m = 0; m < n_mel; m++) { + const double left = mel_lo + m * delta; + const double center = left + delta; + const double right = center + delta; + for (int i = 0; i < n_bins - 1; i++) { + const double f = mel(bin_hz * i); + if (f > left && f < right) { + const double w = f <= center ? (f - left) / (center - left) + : (right - f) / (right - center); + filters.data[(size_t) m * n_bins + i] = (float) w; + } + } + } + } + + // s3 tokenizer: librosa slaney mel, 128 bands + s3tok_cache.fill_sin_cos_table(400); + s3tok_cache.fill_hann_window(400, true); + s3tok_cache.fill_mel_filterbank_matrix(128, 400, 16000); + + // voice encoder: slaney mel, 40 bands up to 8 kHz + ve_cache.fill_sin_cos_table(400); + ve_cache.fill_hann_window(400, true); + ve_cache.fill_mel_filterbank_matrix(40, 400, 16000, 0.0f, 8000.0f); + + // s3gen prompt features at the 24 kHz decoder rate + matcha_cache.fill_sin_cos_table(1920); + matcha_cache.fill_hann_window(1920, true); + matcha_cache.fill_mel_filterbank_matrix(80, 1920, 24000, 0.0f, 8000.0f); +} + +bool mtmd_audio_preprocessor_chatterbox_ref::preprocess(const float * samples, + size_t n_samples, + std::vector & output) const { + output.clear(); + const int sr = (int) hparams.audio_sample_rate; + + // the turbo variant loudness-normalizes the whole clip before any feature + std::vector pcm(samples, samples + n_samples); + if (!is_mtl) { + const float lufs = mtmd_audio_lufs(pcm.data(), pcm.size(), sr); + if (lufs != -HUGE_VALF) { + const float gain = powf(10.0f, (-27.0f - lufs) / 20.0f); + if (std::isfinite(gain) && gain > 0.0f) { + for (float & v : pcm) { + v *= gain; + } + } + } + } + + // entry 0: torchaudio kaldi.fbank of CAMPPlus (snip edges, dc removal, preemphasis 0.97), mean subtracted over time + { + const int frame_len = 400; + const int hop = 160; + const int n_fft = 512; + const int n_mel = hparams.n_mel_bins; + + if ((int) pcm.size() < frame_len) { + return false; + } + const int n_frames = 1 + ((int) pcm.size() - frame_len) / hop; + + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = n_mel; + out.data.assign((size_t) n_mel * n_frames, 0.0f); + + const std::vector & win = fbank_cache.hann_window; + std::vector fft_in((size_t) n_fft * 2), fft_out((size_t) n_fft * 8); + std::vector e; + for (int fr = 0; fr < n_frames; fr++) { + const float * x = pcm.data() + (size_t) fr * hop; + + double mean = 0.0; + for (int i = 0; i < frame_len; i++) { + mean += x[i]; + } + mean /= frame_len; + + std::fill(fft_in.begin(), fft_in.end(), 0.0f); + fft_in[0] = (float) ((x[0] - mean) * (1.0 - 0.97) * win[0]); + for (int i = 1; i < frame_len; i++) { + fft_in[(size_t) i] = (float) (((x[i] - mean) - 0.97 * (x[i - 1] - mean)) * win[(size_t) i]); + } + mtmd_audio_mel_energies(fbank_cache, fft_in, fft_out, false, e); + for (int m = 0; m < n_mel; m++) { + out.data[(size_t) m * n_frames + fr] = (float) log(std::max(e[(size_t) m], (double) FLT_EPSILON)); + } + } + + for (int m = 0; m < n_mel; m++) { + float * row = out.data.data() + (size_t) m * n_frames; + double mean = 0.0; + for (int fr = 0; fr < n_frames; fr++) { + mean += row[fr]; + } + mean /= n_frames; + for (int fr = 0; fr < n_frames; fr++) { + row[fr] -= (float) mean; + } + } + + output.push_back(std::move(out)); + } + + // entries 1 and 2: s3 tokenizer log-mels at the flow and t3 caps, padded to whole 40 ms tokens + auto s3tok_entry = [&](size_t cap) -> bool { + std::vector clip(pcm.begin(), pcm.begin() + std::min(pcm.size(), cap)); + clip.resize((clip.size() + 639) / 640 * 640, 0.0f); + std::vector mel; + int n_frames = 0; + if (!mtmd_audio_s3tok_log_mel(s3tok_cache, clip.data(), clip.size(), mel, n_frames)) { + return false; + } + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = s3tok_cache.filters.n_mel; + out.data = std::move(mel); + output.push_back(std::move(out)); + return true; + }; + const size_t gen_cap = (size_t) 10 * sr; + const size_t t3_cap = (size_t) (is_mtl ? 6 : 15) * sr; + if (!s3tok_entry(gen_cap) || !s3tok_entry(t3_cap)) { + return false; + } + + // entry 3: s3gen prompt features of the flow-capped clip at 24 kHz + { + std::vector clip(pcm.begin(), pcm.begin() + std::min(pcm.size(), gen_cap)); + clip.resize((clip.size() + 639) / 640 * 640, 0.0f); + std::vector pcm24; + mtmd_audio_upsample_3_2(clip.data(), clip.size(), pcm24); + std::vector feat; + int n_frames = 0; + if (!mtmd_audio_matcha_log_mel(matcha_cache, pcm24.data(), pcm24.size(), feat, n_frames)) { + return false; + } + mtmd_audio_mel out; + out.n_len = n_frames; + out.n_len_org = n_frames; + out.n_mel = 80; + out.data = std::move(feat); + output.push_back(std::move(out)); + } + + // entry 4: voice encoder mel of the silence-trimmed clip, padded to the 160-frame partial grid (rate 1.3) + { + size_t t0 = 0, t1 = 0; + mtmd_audio_trim_silence(pcm.data(), pcm.size(), 20.0f, t0, t1); + if (t1 <= t0) { + return false; + } + std::vector mel; + int n_frames = 0; + if (!mtmd_audio_ve_mel(ve_cache, pcm.data() + t0, t1 - t0, mel, n_frames)) { + return false; + } + const int n_mel = 40; + const int n_partial = 160; + const int step = (int) lround((16000.0 / 1.3) / n_partial); + int n_wins = std::max(n_frames - n_partial + step, 0) / step; + const int rem = std::max(n_frames - n_partial + step, 0) % step; + if (n_wins == 0 || (double) (rem + n_partial - step) / n_partial >= 0.8) { + n_wins++; + } + const int target = n_partial + step * (n_wins - 1); + mel.resize((size_t) target * n_mel, 0.0f); + mtmd_audio_mel out; + out.n_len = target; + out.n_len_org = target; + out.n_mel = n_mel; + out.data = std::move(mel); + output.push_back(std::move(out)); + } + return true; +} + // // mtmd_audio_preprocessor_conformer // diff --git a/tools/mtmd/mtmd-audio.h b/tools/mtmd/mtmd-audio.h index 087e4b8c0393..ca95b217f06a 100644 --- a/tools/mtmd/mtmd-audio.h +++ b/tools/mtmd/mtmd-audio.h @@ -171,6 +171,26 @@ struct mtmd_audio_preprocessor_pockettts : mtmd_audio_preprocessor { bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; }; +// chatterbox reference front-end: one clip gives five entries, used by position in mtmd.cpp +// 0: CAMPPlus kaldi fbank [80 x n_frames] (mel major) +// 1: s3 tokenizer log-mel, flow cap [n_mel x n_frames] (mel major) +// 2: s3 tokenizer log-mel, t3 cap [n_mel x n_frames] (mel major) +// 3: s3gen prompt features [n_frames x 80] (frame major, 24 kHz mel rate) +// 4: voice encoder power mel [n_frames x 40] (frame major, trimmed, partial grid) +// turbo loudness-normalizes the clip to -27 LUFS and caps the t3 clip at 15 s instead of 6 s +struct mtmd_audio_preprocessor_chatterbox_ref : mtmd_audio_preprocessor { + mtmd_audio_preprocessor_chatterbox_ref(const clip_ctx * ctx, bool is_mtl) : mtmd_audio_preprocessor(ctx), is_mtl(is_mtl) {} + void initialize() override; + bool preprocess(const float * samples, size_t n_samples, std::vector & output) const override; + + private: + bool is_mtl; + mtmd_audio_cache fbank_cache; // entry 0, the window slot holds the povey window + mtmd_audio_cache s3tok_cache; // entries 1 and 2 + mtmd_audio_cache matcha_cache; // entry 3 + mtmd_audio_cache ve_cache; // entry 4 +}; + struct mtmd_audio_preprocessor_parakeet : mtmd_audio_preprocessor { mtmd_audio_preprocessor_parakeet(clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) { } void initialize() override; diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp index bed8f3d8c6db..0d7901575db0 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -124,6 +124,39 @@ class mtmd_gen_audio_pipeline { return true; } + // encodes a speaker clip or a voice, out receives the encoder embeddings + bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { + if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { + LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); + return false; + } + const std::string marker = mtmd_get_marker(mctx); + mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; + mtmd_input_chunks * chunks = mtmd_input_chunks_init(); + const mtmd_bitmap * bptr = bitmap; + bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; + if (ok) { + ok = false; + for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { + const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); + if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { + continue; + } + if (mtmd_encode_chunk(mctx, chunk) != 0) { + LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); + break; + } + const float * embd = mtmd_get_output_embd(mctx); + const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); + out.assign(embd, embd + n); + ok = true; + break; + } + } + mtmd_input_chunks_free(chunks); + return ok; + } + llama_context * lctx; mtmd_context * mctx; const llama_model * model; @@ -428,39 +461,6 @@ class qwen3tts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { return true; } - // runs the reference wav through the speaker encoder, returns one x-vector embedding row - bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { - LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); - return false; - } - const std::string marker = mtmd_get_marker(mctx); - mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - const mtmd_bitmap * bptr = bitmap; - bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; - if (ok) { - ok = false; - for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { - const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); - if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { - continue; - } - if (mtmd_encode_chunk(mctx, chunk) != 0) { - LOG_ERR("mtmd_helper_gen_audio: speaker encode failed\n"); - break; - } - const float * embd = mtmd_get_output_embd(mctx); - const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); - out.assign(embd, embd + n); - ok = true; - break; - } - } - mtmd_input_chunks_free(chunks); - return ok; - } - // one GEN_WAV process() call over the buffered codes, state is carried across batches bool flush_gen_wav() { if (codes_buf.empty()) { @@ -996,39 +996,6 @@ class pockettts_gen_audio_pipeline : public mtmd_gen_audio_pipeline { return n; } - // runs the reference wav through the mimi encoder, returns one row per 12.5Hz frame - bool encode_speaker(mtmd_bitmap * bitmap, std::vector & out) { - if (!mtmd_bitmap_is_voice(bitmap) && !mtmd_support_audio(mctx)) { - LOG_ERR("mtmd_helper_gen_audio: mmproj has no voice encoder\n"); - return false; - } - const std::string marker = mtmd_get_marker(mctx); - mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - const mtmd_bitmap * bptr = bitmap; - bool ok = mtmd_tokenize(mctx, chunks, &text, &bptr, 1) == 0; - if (ok) { - ok = false; - for (size_t i = 0; i < mtmd_input_chunks_size(chunks); i++) { - const mtmd_input_chunk * chunk = mtmd_input_chunks_get(chunks, i); - if (mtmd_input_chunk_get_type(chunk) != MTMD_INPUT_CHUNK_TYPE_AUDIO) { - continue; - } - if (mtmd_encode_chunk(mctx, chunk) != 0) { - LOG_ERR("mtmd_helper_gen_audio: voice encode failed\n"); - break; - } - const float * embd = mtmd_get_output_embd(mctx); - const size_t n = (size_t) llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk); - out.assign(embd, embd + n); - ok = true; - break; - } - } - mtmd_input_chunks_free(chunks); - return ok; - } - // decodes the buffered latents, the mimi decoder state carries over between calls bool flush_gen_wav() { if (feats_buf.empty()) { @@ -1718,6 +1685,398 @@ class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline { bool wav_header_sent = false; }; +// Chatterbox: the backbone emits s3 speech tokens, then one flow-matching mel decode and the NSF-iSTFT vocoder make the PCM +// the multilingual variant runs cfg with an uncond sequence (text rows zeroed), paired as seq_id and n_seq_max / 2 + seq_id +class chatterbox_gen_audio_pipeline : public mtmd_gen_audio_pipeline { +public: + using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline; + + void reset() override { + seq_id = 0; + seq_uncond = 0; + pos = 0; + ar_idx = 0; + codes_buf.clear(); + audio_pcm.clear(); + h_state_buf.clear(); + out_buf.clear(); + prompt_embd_buf.clear(); + uncond_embd_buf.clear(); + cond_logits.clear(); + n_prompt = 0; + prompt_pos = 0; + decoded = false; + stream = false; + pcm_sent = 0; + wav_header_sent = false; + gen_state.reset(); + } + + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { + reset(); + seq_id = inp->seq_id; + + if (!ensure_cache()) { + return 1; + } + + out_type = inp->out_type; + stream = inp->stream; + if (!init_gen_state(inp->seed)) { + return 1; + } + + if (mtl) { + // the uncond sequence lives half a seq space away from the cond one + const int32_t n_seq_max = (int32_t) llama_n_seq_max(lctx); + seq_uncond = (llama_seq_id) (n_seq_max / 2 + seq_id); + if (n_seq_max < 2 || seq_id >= n_seq_max / 2) { + LOG_ERR("mtmd_helper_gen_audio: multilingual chatterbox needs a paired cfg sequence " + "(n_seq_max = %d, seq_id = %d)\n", n_seq_max, seq_id); + return 1; + } + } + llama_memory_t mem = llama_get_memory(lctx); + if (!llama_memory_seq_rm(mem, seq_id, 0, -1) || (mtl && !llama_memory_seq_rm(mem, seq_uncond, 0, -1))) { + LOG_ERR("mtmd_helper_gen_audio: cannot reset chatterbox sequences\n"); + return 1; + } + + // the encode gives the talker conditioning rows, set_ref keeps the flow decoder reference + if (!inp->speaker_ref) { + LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a speaker reference or a voice\n"); + return 1; + } + if (!encode_speaker(inp->speaker_ref, prompt_embd_buf) || + mtmd_gen_state_set_ref(mctx, gen_state.get()) != 0) { + LOG_ERR("mtmd_helper_gen_audio: speaker reference encoding failed\n"); + return 1; + } + + const int n_e = n_embd; + auto push_row = [&](const float * src) { + prompt_embd_buf.insert(prompt_embd_buf.end(), src, src + n_e); + }; + auto add_pos = [&](float * r, const std::vector & tab, int idx) { + const float * p = tab.data() + (size_t) idx * n_e; + for (int j = 0; j < n_e; j++) { + r[j] += p[j]; + } + }; + + // prompt rows: [conditioning, text, speech bos] + // no text preprocessing here, the [SPACE] substitution and the case folding are baked in at conversion + std::string txt(inp->prompt, inp->prompt_len); + if (mtl && inp->lang && inp->lang[0]) { + // the reference prefixes the language tag, e.g. "[fr]" + std::string lang = inp->lang; + std::transform(lang.begin(), lang.end(), lang.begin(), [](unsigned char c) { return (char) std::tolower(c); }); + txt = "[" + lang + "]" + txt; + } + std::vector ids(txt.size() + 16); + int n_ids = llama_tokenize(vocab, txt.c_str(), (int32_t) txt.size(), ids.data(), (int32_t) ids.size(), + false, true); + if (n_ids < 1) { + LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n"); + return 1; + } + ids.resize((size_t) n_ids); + if (mtl) { + // only the multilingual prompt wraps the text in start/stop tokens + ids.insert(ids.begin(), text_start); + ids.push_back(text_stop); + if (ids.size() > text_pos.size() / (size_t) n_e) { + LOG_ERR("mtmd_helper_gen_audio: prompt too long (%zu tokens, max %zu)\n", ids.size(), text_pos.size() / (size_t) n_e); + return 1; + } + } + const size_t text0 = prompt_embd_buf.size() / (size_t) n_e; + for (size_t i = 0; i < ids.size(); i++) { + push_row(tok_embd.data() + (size_t) ids[i] * n_e); + if (mtl) { + add_pos(prompt_embd_buf.data() + (text0 + i) * n_e, text_pos, (int) i); + } + } + + // speech bos opens the AR stream + push_row(tok_embd.data() + (size_t) llama_vocab_bos(vocab) * n_e); + if (mtl) { + add_pos(prompt_embd_buf.data() + prompt_embd_buf.size() - n_e, speech_pos, 0); + } + ar_idx = 1; + n_prompt = (int) (prompt_embd_buf.size() / (size_t) n_e); + + if (mtl) { + // cfg uncond prompt: text rows are zeroed but keep their learned positions, as in the reference + uncond_embd_buf = prompt_embd_buf; + for (size_t i = 0; i < ids.size(); i++) { + float * r = uncond_embd_buf.data() + (text0 + i) * n_e; + std::fill(r, r + n_e, 0.0f); + add_pos(r, text_pos, (int) i); + } + } + + prompt_pos = 0; + return 0; + } + + int32_t step_prompt(int32_t n_batch) override { + GGML_ASSERT(n_batch > 0); + // the uncond prompt follows the cond one on mtl + const int n_total = mtl ? 2 * n_prompt : n_prompt; + if (prompt_pos >= n_total) { + return 0; + } + const bool uncond = prompt_pos >= n_prompt; + const int p0 = uncond ? prompt_pos - n_prompt : prompt_pos; + const int32_t n_tokens_batch = std::min(n_batch, n_prompt - p0); + const bool last = p0 + n_tokens_batch == n_prompt; + + decode_embd_batch batch((uncond ? uncond_embd_buf : prompt_embd_buf).data(), n_prompt, 1, n_embd); + batch.set_position_normal(0, uncond ? seq_uncond : seq_id); + if (last) { + batch.logits[n_prompt - 1] = 1; + } + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, p0, n_tokens_batch)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n"); + return -1; + } + prompt_pos += n_tokens_batch; + + if (last && mtl) { + if (!uncond) { + cfg_read(); + } else { + cfg_apply(); + } + } + + if (prompt_pos >= n_total) { + pos = n_prompt; + prompt_embd_buf.clear(); + uncond_embd_buf.clear(); + return 0; + } + return n_total - prompt_pos; + } + + int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override { + GGML_UNUSED(h_state_in); + if (sampled < 0 || sampled >= llama_vocab_n_tokens(vocab)) { + LOG_ERR("mtmd_helper_gen_audio: chatterbox requires a token sampled from the backbone\n"); + return 1; + } + + // the speech stop token is the backbone eos, the multilingual speech positions end the stream too + if (llama_vocab_is_eog(vocab, sampled) || (mtl && (size_t) ar_idx >= speech_pos.size() / (size_t) n_embd)) { + *out_stop = true; + *h_state_out = nullptr; + return 0; + } + + // keep only the 6561 s3gen codes, dropping start/stop and oov ids + if (sampled >= speech_base && sampled - speech_base < 6561) { + codes_buf.push_back(sampled - speech_base); + } + + std::vector e(tok_embd.begin() + (size_t) sampled * n_embd, + tok_embd.begin() + (size_t) (sampled + 1) * n_embd); + if (mtl) { + // the multilingual backbone takes the token row plus the learned speech position + const float * p = speech_pos.data() + (size_t) ar_idx * n_embd; + for (int j = 0; j < n_embd; j++) { + e[(size_t) j] += p[j]; + } + ar_idx++; + } + + if (!decode_one(e.data(), seq_id)) { + return 1; + } + if (mtl) { + // cfg: the sampled token feeds both branches + cfg_read(); + if (!decode_one(e.data(), seq_uncond)) { + return 1; + } + cfg_apply(); + } + pos++; + + const float * h = llama_get_embeddings_ith(lctx, -1); + h_state_buf.assign(h, h + n_embd); + *h_state_out = h_state_buf.data(); + return 0; + } + + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { + *out_sample_rate = info.sample_rate; + + if (!stream) { + if (!flush_gen_wav()) { + return 1; + } + if (out_n_samples) { + *out_n_samples = (int64_t) audio_pcm.size(); + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) audio_pcm.data(); + *out_data_len = audio_pcm.size() * sizeof(float); + return 0; + } + out_buf.clear(); + if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { + LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); + return 1; + } + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + // streaming: the flow decode is one-shot, so all the audio arrives after flush() + const size_t n_new = audio_pcm.size() - pcm_sent; + if (out_n_samples) { + *out_n_samples = (int64_t) n_new; + } + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { + *out_data = (const char *) (audio_pcm.data() + pcm_sent); + *out_data_len = n_new * sizeof(float); + pcm_sent = audio_pcm.size(); + return 0; + } + out_buf.clear(); + if (!wav_header_sent) { + write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); + wav_header_sent = true; + } + append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); + pcm_sent = audio_pcm.size(); + *out_data = out_buf.data(); + *out_data_len = out_buf.size(); + return 0; + } + + int32_t flush() override { + return flush_gen_wav() ? 0 : 1; + } + +private: + bool decode_one(const float * e, llama_seq_id seq) { + decode_embd_batch batch(e, 1, 1, n_embd); + batch.set_position_normal(pos, seq); + batch.logits[0] = 1; + if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.render(lctx, 0, 1)) != 0) { + LOG_ERR("mtmd_helper_gen_audio: step decode failed\n"); + return false; + } + return true; + } + + // cfg of the reference: logits = cond + 0.5 * (cond - uncond) + // the mix goes into the last output row (the uncond one), the caller samples from it + void cfg_read() { + const float * c = llama_get_logits_ith(lctx, -1); + cond_logits.assign(c, c + llama_vocab_n_tokens(vocab)); + } + void cfg_apply() { + float * u = llama_get_logits_ith(lctx, -1); + for (size_t i = 0; i < cond_logits.size(); i++) { + u[i] = cond_logits[i] + 0.5f * (cond_logits[i] - u[i]); + } + } + + // one GEN_WAV call over the whole utterance, the flow decode is not streamed + bool flush_gen_wav() { + if (decoded || codes_buf.empty()) { + return true; + } + if (!mtl) { + // turbo appends a short silence tail before vocoding + codes_buf.insert(codes_buf.end(), 3, 4299); + } + mtmd_gen_inp inp = mtmd_gen_inp_default(mctx); + inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV; + inp.codes = codes_buf.data(); + inp.n_codes = codes_buf.size(); + mtmd_gen_out out{}; + if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) { + LOG_ERR("mtmd_helper_gen_audio: gen_wav decode failed\n"); + return false; + } + audio_pcm.assign(out.audio, out.audio + out.n_samples); + decoded = true; + return true; + } + + bool ensure_cache() { + if (!tok_embd.empty()) { + return true; + } + // fused vocab layout: [text 0..speech_base) then the speech tokens + speech_base = find_special_token(vocab, "<|speech_0|>"); + if (speech_base == LLAMA_TOKEN_NULL) { + LOG_ERR("mtmd_helper_gen_audio: fused speech tokens not found in vocab\n"); + return false; + } + + const uint32_t n_tok_embd = llama_model_get_tok_embd(model, nullptr); + if (n_tok_embd != (uint32_t) llama_vocab_n_tokens(vocab) * (uint32_t) n_embd) { + LOG_ERR("mtmd_helper_gen_audio: unexpected token embedding size\n"); + return false; + } + tok_embd.resize(n_tok_embd); + llama_model_get_tok_embd(model, tok_embd.data()); + + auto read_table = [&](const char * name, std::vector & dst) { + const size_t n = mtmd_gen_audio_read_tensor(mctx, name, nullptr, 0); + dst.resize(n); + if (n == 0 || mtmd_gen_audio_read_tensor(mctx, name, dst.data(), n) != n) { + LOG_ERR("mtmd_helper_gen_audio: %s read failed\n", name); + return false; + } + return true; + }; + + // multilingual: the learned position tables + mtl = mtmd_gen_audio_read_tensor(mctx, "a.gen.t3.text_pos_emb", nullptr, 0) > 0; + if (mtl && (!read_table("a.gen.t3.text_pos_emb", text_pos) || !read_table("a.gen.t3.speech_pos_emb", speech_pos))) { + return false; + } + return true; + } + + // vocab and mmproj data fixed across the whole session, read once + std::vector tok_embd; + bool mtl = false; + std::vector text_pos; + std::vector speech_pos; + llama_token speech_base = LLAMA_TOKEN_NULL; + // start_text_token and stop_text_token of the reference, only the multilingual prompt uses them + static constexpr llama_token text_start = 255; + static constexpr llama_token text_stop = 0; + + // per-generation state, cleared by reset() + llama_seq_id seq_id = 0; + llama_seq_id seq_uncond = 0; + llama_pos pos = 0; + int ar_idx = 0; + std::vector prompt_embd_buf; + std::vector uncond_embd_buf; + int n_prompt = 0; + int prompt_pos = 0; + std::vector cond_logits; + std::vector codes_buf; + bool decoded = false; + std::vector audio_pcm; + std::vector h_state_buf; + mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; + std::vector out_buf; + bool stream = false; + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; +}; + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { switch (mtmd_gen_audio_get_info(mctx).type) { case MTMD_GEN_AUDIO_TYPE_QWEN3TTS: @@ -1728,6 +2087,8 @@ static std::unique_ptr make_pipeline(llama_context * lc return std::unique_ptr(new soprano_gen_audio_pipeline(lctx, mctx)); case MTMD_GEN_AUDIO_TYPE_LFM3A: return std::unique_ptr(new lfm3a_gen_audio_pipeline(lctx, mctx)); + case MTMD_GEN_AUDIO_TYPE_CHATTERBOX: + return std::unique_ptr(new chatterbox_gen_audio_pipeline(lctx, mctx)); default: return nullptr; } diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index c1b024a8840c..1c89eabfa355 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -176,6 +176,7 @@ struct mtmd_bitmap { std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking bool is_audio = false; // true if the bitmap is audio bool is_voice = false; + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any bool mergeable = false; // [QWEN_VIDEO] set only on frames of the same video bool is_video = false; // see mtmd_bitmap_set_is_video() @@ -201,9 +202,14 @@ struct mtmd_bitmap { } } - static mtmd_bitmap * init_voice(const std::string & name, const float * rows, uint32_t n_embd, uint32_t n_rows) { + static mtmd_bitmap * init_voice(const std::string & name, const clip_gen_voice & voice, uint32_t n_embd) { + const float * rows = voice.rows.data(); + const uint32_t n_rows = voice.rows.size() / n_embd; auto * bitmap = new mtmd_bitmap(nullptr, n_embd, n_rows); bitmap->is_voice = true; + if (!voice.ref.tokens.empty()) { + bitmap->gen_ref = std::make_shared(voice.ref); + } bitmap->id = name; bitmap->data.assign((const unsigned char *) rows, (const unsigned char *) (rows + (size_t) n_embd * n_rows)); return bitmap; @@ -348,6 +354,7 @@ struct mtmd_audio_track { clip_image_f32_batch batch_f32; // the mel of the whole track uint32_t n_tokens = 0; std::vector embd; // filled by the first encode + std::shared_ptr gen_ref; // gen-side reference of a predefined voice, if any }; struct mtmd_audio_tokens { @@ -564,6 +571,11 @@ mtmd_context_params mtmd_context_params_default() { return params; } +// chatterbox multilingual has a perceiver in its conditioning encoder (speaker encoding context) +static bool cbx_is_mtl(clip_ctx * ctx_a) { + return clip_cbx_read_tensor(ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) > 0; +} + struct mtmd_context { struct clip_ctx * ctx_v; // vision struct clip_ctx * ctx_a; // audio @@ -572,6 +584,9 @@ struct mtmd_context { // generation context struct clip_ctx * ctx_gen_a; // audio + // gen-side reference data of the last encoded chunk (chatterbox), see mtmd_gen_state_set_ref() + clip_gen_ref gen_ref; + bool print_timings; int n_threads; std::string media_marker; @@ -1184,6 +1199,10 @@ struct mtmd_context { { audio_preproc = std::make_unique(ctx_a); } break; + case PROJECTOR_TYPE_CHATTERBOX_SPKENC: + { + audio_preproc = std::make_unique(ctx_a, cbx_is_mtl(ctx_a)); + } break; default: throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj)); } @@ -1845,6 +1864,38 @@ struct mtmd_tokenizer { } } + // one chunk for the five entries of a chatterbox clip, one token per talker conditioning row + if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + GGML_ASSERT(mel_spec_chunks.size() == 5); + // mtl: spkr, perceiver query rows, emotion; turbo: spkr, t3 speech tokens + const size_t n_query = clip_cbx_read_tensor(ctx->ctx_a, "a.cenc.perceiver.pre_attention_query", nullptr, 0) / ctx->n_embd_out(); + const size_t n_tokens = cbx_is_mtl(ctx->ctx_a) ? 2 + n_query : 1 + (size_t) mel_spec_chunks[2].n_len / 4; + + clip_image_f32_batch batch_f32; + batch_f32.is_audio = true; + for (auto & mel_spec : mel_spec_chunks) { + GGML_ASSERT(mel_spec.n_len <= INT32_MAX && mel_spec.n_len >= 0); + GGML_ASSERT(mel_spec.n_mel <= INT32_MAX && mel_spec.n_mel >= 0); + clip_image_f32 mel_f32; + mel_f32.set_size({(int) mel_spec.n_len, (int) mel_spec.n_mel}, + mel_spec.data.empty(), /* is_audio */ true); + mel_f32.cpy_buf(mel_spec.data); + batch_f32.entries.push_back(std::move(mel_f32)); + } + + mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); + audio_tokens->n_tokens = (uint32_t) n_tokens; + audio_tokens->batch_f32 = std::move(batch_f32); + audio_tokens->id = bitmap->id; + + mtmd_input_chunk chunk{ + MTMD_INPUT_CHUNK_TYPE_AUDIO, + {}, // text tokens + nullptr, // image tokens + std::move(audio_tokens), + }; + cur.entries.emplace_back(std::move(chunk)); + } else // consider each mel_spec as a separate audio chunk // TODO: maybe support batching, but this may come with memory cost for (auto & mel_spec : mel_spec_chunks) { @@ -1898,6 +1949,7 @@ struct mtmd_tokenizer { track->n_tokens = bitmap.ny; const auto & buf = bitmap.get_ro_buf(); track->embd.assign((const float *) buf.data(), (const float *) (buf.data() + buf.size())); + track->gen_ref = bitmap.gen_ref; mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens); audio_tokens->n_tokens = bitmap.ny; @@ -2189,7 +2241,10 @@ static int32_t mtmd_encode_impl(mtmd_context * ctx, const mtmd_image_tokens * im return ok ? 0 : 1; } +static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd); + static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk * chunk, std::vector & out_embd) { + ctx->gen_ref = {}; // see mtmd_gen_state_set_ref() if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) { LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n"); return 0; @@ -2234,8 +2289,14 @@ static int32_t mtmd_encode_chunk_impl(mtmd_context * ctx, const mtmd_input_chunk const size_t off = (size_t) chunk->tokens_audio->row_begin * n_mmproj_embd; out_embd.assign(track->embd.begin() + off, track->embd.begin() + off + (size_t) chunk->tokens_audio->n_tokens * n_mmproj_embd); + if (track->gen_ref) { + ctx->gen_ref = *track->gen_ref; + } return 0; } + if (clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_CHATTERBOX_SPKENC) { + return cbx_ref_encode(ctx, chunk->tokens_audio.get(), out_embd); + } out_embd.resize((size_t)chunk->tokens_audio->n_tokens * ctx->n_embd_out()); bool ok = clip_image_batch_encode( ctx->ctx_a, @@ -2302,6 +2363,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { info.type = MTMD_GEN_AUDIO_TYPE_LFM3A; info.sample_rate = 24000; break; + case PROJECTOR_TYPE_CHATTERBOX: + info.type = MTMD_GEN_AUDIO_TYPE_CHATTERBOX; + info.sample_rate = 24000; + break; default: info.type = MTMD_GEN_AUDIO_TYPE_NONE; break; @@ -2309,6 +2374,355 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) { return info; } +size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max) { + if (!ctx->ctx_gen_a) { + return 0; + } + return clip_cbx_read_tensor(ctx->ctx_gen_a, name, out, n_max); +} + +// raw 192-dim CAMPPlus x-vector of the fbank entry +static bool cbx_ref_xvec(mtmd_context * ctx, const clip_image_f32 & fbank, std::vector & xvec) { + clip_image_f32 mel_img; + mel_img.set_size(fbank.get_size(), false, true); + mel_img.cpy_buf(fbank.get_ro_buf()); + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(mel_img)); + xvec.resize(192); + if (!clip_image_batch_encode(ctx->ctx_a, ctx->n_threads, &batch, xvec)) { + LOG_ERR("%s: speaker encoder failed\n", __func__); + return false; + } + return true; +} + +// s3 speech tokens of a log-mel entry, rows optionally receives their speech embedding rows +static int32_t cbx_ref_tokenize(mtmd_context * ctx, const clip_image_f32 & mel, + std::vector & codes, std::vector * rows) { + clip_image_f32 mel_img; + mel_img.set_size(mel.get_size(), false, true); + mel_img.cpy_buf(mel.get_ro_buf()); + + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(mel_img)); + + std::vector out_codes; + std::vector out_embd; + + clip_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = CLIP_GEN_PROCESS_TOKENIZE; + params.out_codes = &out_codes; + params.out_code_embd = rows ? &out_embd : nullptr; + + if (!clip_encode(ctx->ctx_gen_a, ¶ms)) { + LOG_ERR("%s: clip_encode failed (tokenize)\n", __func__); + return 1; + } + + codes = std::move(out_codes); + if (rows) { + *rows = std::move(out_embd); + } + return 0; +} + +// talker conditioning rows: the speaker row, then on mtl the perceiver block and the emotion row +static int32_t cbx_ref_cond(mtmd_context * ctx, const clip_image_f32 & ve_mel, + const std::vector & pse, std::vector & out_rows) { + clip_ctx * ctx_clip = ctx->ctx_a; + // n_expect 0 accepts any non-empty size + auto read_t = [&](const char * name, std::vector & v, size_t n_expect) -> bool { + const size_t n = clip_cbx_read_tensor(ctx_clip, name, nullptr, 0); + if (n == 0 || (n_expect > 0 && n != n_expect)) { + return false; + } + v.resize(n); + return clip_cbx_read_tensor(ctx_clip, name, v.data(), n) == n; + }; + + // voice encoder (embeds_from_wavs): 3-layer lstm per 160-frame partial, the normalized partial embeddings are averaged + const std::vector & mel = ve_mel.get_ro_buf(); + const int n_mel = 40; + const int n_partial = 160; + const int n_h = 256; + const int step = (int) lround((16000.0 / 1.3) / n_partial); // reference rate 1.3 + if (ve_mel.nx() < n_partial || mel.size() < (size_t) ve_mel.nx() * n_mel) { + LOG_ERR("%s: voice encoder mel too short\n", __func__); + return 1; + } + const int n_wins = (ve_mel.nx() - n_partial) / step + 1; + + std::vector w_ih[3], w_hh[3], b_ih[3], b_hh[3]; + std::vector w_proj, b_proj; + for (int l = 0; l < 3; l++) { + const std::string s = std::to_string(l); + const size_t n_in = l == 0 ? n_mel : n_h; + if (!read_t(("a.ve.lstm.weight_ih_l" + s).c_str(), w_ih[l], 4 * n_h * n_in) || + !read_t(("a.ve.lstm.weight_hh_l" + s).c_str(), w_hh[l], 4 * n_h * n_h) || + !read_t(("a.ve.lstm.bias_ih_l" + s).c_str(), b_ih[l], 4 * n_h) || + !read_t(("a.ve.lstm.bias_hh_l" + s).c_str(), b_hh[l], 4 * n_h)) { + LOG_ERR("%s: model has no voice encoder\n", __func__); + return 1; + } + } + if (!read_t("a.ve.proj.weight", w_proj, n_h * n_h) || !read_t("a.ve.proj.bias", b_proj, n_h)) { + LOG_ERR("%s: model has no voice encoder projection\n", __func__); + return 1; + } + + std::vector ve(n_h, 0.0f); + std::vector h((size_t) 3 * n_h), c((size_t) 3 * n_h), x(n_h), g((size_t) 4 * n_h); + for (int p = 0; p < n_wins; p++) { + std::fill(h.begin(), h.end(), 0.0f); + std::fill(c.begin(), c.end(), 0.0f); + for (int t = 0; t < n_partial; t++) { + const float * in = mel.data() + (size_t) (p * step + t) * n_mel; + int n_in = n_mel; + for (int l = 0; l < 3; l++) { + float * hl = h.data() + (size_t) l * n_h; + float * cl = c.data() + (size_t) l * n_h; + for (int j = 0; j < 4 * n_h; j++) { + double acc = b_ih[l][(size_t) j] + b_hh[l][(size_t) j]; + const float * wi = w_ih[l].data() + (size_t) j * n_in; + for (int i = 0; i < n_in; i++) { + acc += (double) wi[i] * in[i]; + } + const float * wh = w_hh[l].data() + (size_t) j * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) wh[i] * hl[i]; + } + g[(size_t) j] = (float) acc; + } + // torch gate order: input, forget, cell, output + for (int i = 0; i < n_h; i++) { + const float gi = 1.0f / (1.0f + expf(-g[(size_t) i])); + const float gf = 1.0f / (1.0f + expf(-g[(size_t) i + n_h])); + const float gc = tanhf(g[(size_t) i + 2 * n_h]); + const float go = 1.0f / (1.0f + expf(-g[(size_t) i + 3 * n_h])); + cl[i] = gf * cl[i] + gi * gc; + x[(size_t) i] = go * tanhf(cl[i]); + } + memcpy(hl, x.data(), (size_t) n_h * sizeof(float)); + in = hl; + n_in = n_h; + } + } + // projected, relu'd, normalized partial embedding + std::vector e(n_h); + double norm = 0.0; + for (int o = 0; o < n_h; o++) { + double acc = b_proj[(size_t) o]; + const float * w = w_proj.data() + (size_t) o * n_h; + const float * hl = h.data() + (size_t) 2 * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) w[i] * hl[i]; + } + e[(size_t) o] = (float) std::max(acc, 0.0); + norm += (double) e[(size_t) o] * e[(size_t) o]; + } + norm = sqrt(norm); + for (int o = 0; o < n_h; o++) { + ve[(size_t) o] += (float) (e[(size_t) o] / norm); + } + } + double norm = 0.0; + for (float v : ve) { + norm += (double) v * v; + } + norm = sqrt(norm); + for (float & v : ve) { + v = (float) (v / norm); + } + + // speaker projection row + const int n_e = (int) ctx->n_embd_out(); + std::vector w_spkr, b_spkr; + if (!read_t("a.cenc.spkr_enc.weight", w_spkr, (size_t) n_e * n_h) || !read_t("a.cenc.spkr_enc.bias", b_spkr, n_e)) { + LOG_ERR("%s: model has no speaker conditioning projection\n", __func__); + return 1; + } + std::vector rows((size_t) n_e); + for (int o = 0; o < n_e; o++) { + double acc = b_spkr[(size_t) o]; + const float * w = w_spkr.data() + (size_t) o * n_h; + for (int i = 0; i < n_h; i++) { + acc += (double) w[i] * ve[(size_t) i]; + } + rows[(size_t) o] = (float) acc; + } + + // the perceiver runs its shared attention block as cross then self attention + std::vector query; + if (read_t("a.cenc.perceiver.pre_attention_query", query, 0)) { + const int n_head = 4; + const size_t n_ee = (size_t) n_e * n_e; + if (pse.empty() || pse.size() % n_e != 0 || query.size() % n_e != 0 || n_e % n_head != 0) { + LOG_ERR("%s: reference speech embeddings required for the perceiver\n", __func__); + return 1; + } + std::vector ln_w, ln_b, wq, bq, wk, bk, wv, bv, wo, bo, emo; + if (!read_t("a.cenc.perceiver.attn.norm.weight", ln_w, n_e) || !read_t("a.cenc.perceiver.attn.norm.bias", ln_b, n_e) || + !read_t("a.cenc.perceiver.attn.to_q.weight", wq, n_ee) || !read_t("a.cenc.perceiver.attn.to_q.bias", bq, n_e) || + !read_t("a.cenc.perceiver.attn.to_k.weight", wk, n_ee) || !read_t("a.cenc.perceiver.attn.to_k.bias", bk, n_e) || + !read_t("a.cenc.perceiver.attn.to_v.weight", wv, n_ee) || !read_t("a.cenc.perceiver.attn.to_v.bias", bv, n_e) || + !read_t("a.cenc.perceiver.attn.proj_out.weight", wo, n_ee) || !read_t("a.cenc.perceiver.attn.proj_out.bias", bo, n_e) || + !read_t("a.cenc.emotion_adv_fc.weight", emo, n_e)) { + LOG_ERR("%s: model has an incomplete perceiver\n", __func__); + return 1; + } + const int d_head = n_e / n_head; + + auto layer_norm = [&](const float * in, float * out) { + double mean = 0.0, var = 0.0; + for (int i = 0; i < n_e; i++) { + mean += in[i]; + } + mean /= n_e; + for (int i = 0; i < n_e; i++) { + var += (in[i] - mean) * (in[i] - mean); + } + const double sd = sqrt(var / n_e + 1e-5); + for (int i = 0; i < n_e; i++) { + out[i] = (float) ((in[i] - mean) / sd * ln_w[(size_t) i] + ln_b[(size_t) i]); + } + }; + auto linear = [&](const std::vector & w, const std::vector & b, + const std::vector & in, int n_rows, std::vector & out) { + out.resize((size_t) n_rows * n_e); + for (int r = 0; r < n_rows; r++) { + for (int o = 0; o < n_e; o++) { + double acc = b[(size_t) o]; + const float * wr = w.data() + (size_t) o * n_e; + const float * ir = in.data() + (size_t) r * n_e; + for (int i = 0; i < n_e; i++) { + acc += (double) wr[i] * ir[i]; + } + out[(size_t) r * n_e + o] = (float) acc; + } + } + }; + auto attn_block = [&](const std::vector & x1, int n1, + const std::vector & x2, int n2, std::vector & out) { + std::vector nx1((size_t) n1 * n_e), nx2((size_t) n2 * n_e); + for (int r = 0; r < n1; r++) { + layer_norm(x1.data() + (size_t) r * n_e, nx1.data() + (size_t) r * n_e); + } + for (int r = 0; r < n2; r++) { + layer_norm(x2.data() + (size_t) r * n_e, nx2.data() + (size_t) r * n_e); + } + std::vector q, k, v; + linear(wq, bq, nx1, n1, q); + linear(wk, bk, nx2, n2, k); + linear(wv, bv, nx2, n2, v); + + std::vector ctxt((size_t) n1 * n_e); + std::vector sc((size_t) n2); + for (int hd = 0; hd < n_head; hd++) { + const int off = hd * d_head; + for (int t = 0; t < n1; t++) { + double mx = -1e30; + for (int s = 0; s < n2; s++) { + double acc = 0.0; + for (int i = 0; i < d_head; i++) { + acc += (double) q[(size_t) t * n_e + off + i] * k[(size_t) s * n_e + off + i]; + } + sc[(size_t) s] = acc / sqrt((double) d_head); + mx = std::max(mx, sc[(size_t) s]); + } + double sum = 0.0; + for (int s = 0; s < n2; s++) { + sc[(size_t) s] = exp(sc[(size_t) s] - mx); + sum += sc[(size_t) s]; + } + for (int i = 0; i < d_head; i++) { + double acc = 0.0; + for (int s = 0; s < n2; s++) { + acc += sc[(size_t) s] * v[(size_t) s * n_e + off + i]; + } + ctxt[(size_t) t * n_e + off + i] = (float) (acc / sum); + } + } + } + linear(wo, bo, ctxt, n1, out); + for (size_t i = 0; i < out.size(); i++) { + out[i] += x1[i]; + } + }; + + const int n_q = (int) (query.size() / n_e); + std::vector pre, p32; + attn_block(query, n_q, pse, (int) (pse.size() / (size_t) n_e), pre); + attn_block(pre, n_q, pre, n_q, p32); + + rows.insert(rows.end(), p32.begin(), p32.end()); + const float exaggeration = 0.5f; // reference default + for (int i = 0; i < n_e; i++) { + rows.push_back(emo[(size_t) i] * exaggeration); + } + } + + out_rows = std::move(rows); + return 0; +} + +// talker conditioning rows go to out_embd, the flow decoder reference to ctx->gen_ref +static int32_t cbx_ref_encode(mtmd_context * ctx, const mtmd_audio_tokens * audio_tokens, std::vector & out_embd) { + if (!ctx->ctx_gen_a) { + LOG_ERR("%s: reference encoding needs the audio generation context\n", __func__); + return 1; + } + const auto & entries = audio_tokens->batch_f32.entries; + GGML_ASSERT(entries.size() == 5); + + std::vector xvec; + if (!cbx_ref_xvec(ctx, entries[0], xvec)) { + return 1; + } + + std::vector flow_codes, t3_codes; + std::vector t3_rows; + if (cbx_ref_tokenize(ctx, entries[1], flow_codes, nullptr) != 0 || + cbx_ref_tokenize(ctx, entries[2], t3_codes, &t3_rows) != 0) { + return 1; + } + + const size_t n_feat = (size_t) entries[3].nx() * 80; + if ((size_t) entries[3].nx() != 2 * flow_codes.size()) { + LOG_ERR("%s: reference mel length %d does not match %zu codes\n", + __func__, entries[3].nx(), flow_codes.size()); + return 1; + } + + // [spkr, perceiver block] on mtl, [spkr, reference speech rows] on turbo + const bool mtl = cbx_is_mtl(ctx->ctx_a); + std::vector pse; + if (mtl) { + pse = std::move(t3_rows); + } + std::vector rows; + if (cbx_ref_cond(ctx, entries[4], pse, rows) != 0) { + return 1; + } + if (!mtl) { + rows.insert(rows.end(), t3_rows.begin(), t3_rows.end()); + } + if (rows.size() != (size_t) audio_tokens->n_tokens * (size_t) ctx->n_embd_out()) { + LOG_ERR("%s: unexpected number of conditioning rows\n", __func__); + return 1; + } + + ctx->gen_ref.tokens = std::move(flow_codes); + ctx->gen_ref.feat.assign(entries[3].get_ro_buf().begin(), entries[3].get_ro_buf().begin() + n_feat); + ctx->gen_ref.spk = std::move(xvec); + + out_embd = std::move(rows); + return 0; +} + const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx) { if (!ctx->ctx_gen_a || idx < 0) { return nullptr; @@ -2330,8 +2744,7 @@ mtmd_bitmap * mtmd_gen_audio_get_voice(const mtmd_context * ctx, const char * na if (it == voices.end()) { return nullptr; } - const uint32_t n_embd = clip_n_mmproj_embd(ctx->ctx_gen_a); - return mtmd_bitmap::init_voice(it->first, it->second.data(), n_embd, it->second.size() / n_embd); + return mtmd_bitmap::init_voice(it->first, it->second, clip_n_mmproj_embd(ctx->ctx_gen_a)); } mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) { @@ -2382,6 +2795,9 @@ struct mtmd_gen_state { std::vector out_embd; // next-step hidden state fed back to backbone (GEN_CODE) std::vector out_audio; // decoded PCM samples for the current frame (GEN_WAV) + // voice reference for GEN_WAV, from a speaker clip or a predefined voice (chatterbox) + clip_gen_ref ref; + // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host std::unique_ptr istft; @@ -2488,6 +2904,19 @@ static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, co return 0; } +int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state) { + if (!state || state->ctx_clip != ctx->ctx_gen_a) { + LOG_ERR("%s: invalid gen state\n", __func__); + return 1; + } + if (ctx->gen_ref.tokens.empty()) { + LOG_ERR("%s: the last encode produced no reference data\n", __func__); + return 1; + } + state->ref = ctx->gen_ref; + return 0; +} + static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) { clip_ctx * ctx_clip = ctx->ctx_gen_a; if (!ctx_clip) { @@ -2569,6 +2998,44 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * } // MTMD_GEN_PROCESS_TYPE_GEN_WAV + if (clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_CHATTERBOX) { + if (!inp->codes || inp->n_codes == 0) { + LOG_ERR("%s: codes required for gen_wav\n", __func__); + return 1; + } + if (state->ref.tokens.empty()) { + LOG_ERR("%s: gen_wav needs a voice reference, see mtmd_gen_state_set_ref()\n", __func__); + return 1; + } + std::vector in_codes(inp->codes, inp->codes + inp->n_codes); + + // the batch entry is unused, present to satisfy the encode interface + clip_image_f32 dummy; + dummy.set_size({1, 1}, false, true); + dummy.cpy_buf(std::vector(1, 0.0f)); + clip_image_f32_batch batch; + batch.is_audio = true; + batch.entries.push_back(std::move(dummy)); + + clip_encode_params params; + params.imgs = &batch; + params.n_threads = ctx->n_threads; + params.gen_process = CLIP_GEN_PROCESS_TTS; + params.codes = &in_codes; + params.out_audio = &state->out_audio; + params.ref = &state->ref; + params.gen_state = state->clip_state; + + if (!clip_encode(ctx_clip, ¶ms)) { + LOG_ERR("%s: clip_encode failed (gen_wav)\n", __func__); + return 1; + } + + out->audio = state->out_audio.data(); + out->n_samples = state->out_audio.size(); + return 0; + } + const bool has_codes = inp->codes && inp->n_codes > 0; const bool has_feats = inp->feats && inp->n_feats > 0; if (has_codes == has_feats) { diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index 4f69a644d857..d490e9668b47 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -465,6 +465,7 @@ enum mtmd_gen_audio_type { MTMD_GEN_AUDIO_TYPE_POCKETTTS, MTMD_GEN_AUDIO_TYPE_SOPRANO, MTMD_GEN_AUDIO_TYPE_LFM3A, + MTMD_GEN_AUDIO_TYPE_CHATTERBOX, }; struct mtmd_gen_audio_info { @@ -476,6 +477,8 @@ struct mtmd_gen_audio_info { MTMD_API struct mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx); +// read a named tensor of the gen audio model as F32, returns the element count (0 if not found), out may be null to query the size +MTMD_API size_t mtmd_gen_audio_read_tensor(mtmd_context * ctx, const char * name, float * out, size_t n_max); // name of the idx-th predefined voice, nullptr when idx is out of range MTMD_API const char * mtmd_gen_audio_voice_name(const mtmd_context * ctx, int32_t idx); @@ -543,6 +546,10 @@ MTMD_API void mtmd_gen_state_free(mtmd_gen_state * state); // clear the decoder state (cold start), keep the RNG stream MTMD_API void mtmd_gen_state_reset(mtmd_gen_state * state); +// copy the gen-side reference data of the last mtmd_encode_chunk() call into the state (speaker clip or predefined voice) +// returns non-zero if the last encode produced no such data +MTMD_API int32_t mtmd_gen_state_set_ref(mtmd_context * ctx, mtmd_gen_state * state); + // defaults tuned for the loaded pipeline, callers override only what they care about MTMD_API struct mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx); diff --git a/tools/tts/README.md b/tools/tts/README.md index 9bec176245fc..82c23473c888 100644 --- a/tools/tts/README.md +++ b/tools/tts/README.md @@ -121,3 +121,36 @@ The mmproj holds the audio encoder (for ASR), the depthformer and the codec deco python convert_hf_to_gguf.py path/to/lfm3-audio --outfile lfm3-audio.gguf python convert_hf_to_gguf.py path/to/lfm3-audio --mmproj --outfile mmproj-lfm3-audio.gguf ``` + +## Chatterbox + +Two variants are supported: [Chatterbox Turbo](https://huggingface.co/ResembleAI/chatterbox-turbo) (English) and [Chatterbox Multilingual V3](https://huggingface.co/ResembleAI/chatterbox) (23 languages). + +Available params: +- `--tts-speaker-file` should point to a speaker reference audio file (wav, mp3). Optional, the predefined voice is used without it +- `--tts-voice` can be `default`, the voice shipped with the model (`conds.pt`) +- `--tts-lang` sets the language of the multilingual variant, for example `fr` (default: none). It is not used by Turbo + +Example usage: + +```sh +llama-tts -m chatterbox-mtl.gguf \ + -mm mmproj-chatterbox-mtl.gguf \ + -p "Bonjour tout le monde" \ + --tts-lang fr \ + --tts-speaker-file speaker.mp3 \ + --output out.wav +``` + +Note: the multilingual variant runs classifier-free guidance with a second sequence. `llama-tts` sets this up by itself, but `llama-server` does not support it yet. + +**Note for GGUF conversion:** + +The upstream repositories ship raw checkpoints without `config.json`. Download the files below into a directory, then convert it as usual (the variant is detected from the talker file): +- Turbo: `t3_turbo_v1.safetensors`, `s3gen_meanflow.safetensors`, `ve.safetensors`, `conds.pt` and the GPT-2 tokenizer files (`vocab.json`, `merges.txt`, `added_tokens.json`, `special_tokens_map.json`, `tokenizer_config.json`) +- Multilingual V3: `t3_mtl23ls_v3.safetensors`, `s3gen_v3.safetensors`, `ve.safetensors`, `conds.pt`, `mtl_tokenizer.json` + +```sh +python convert_hf_to_gguf.py path/to/chatterbox --outfile chatterbox.gguf +python convert_hf_to_gguf.py path/to/chatterbox --mmproj --outfile mmproj-chatterbox.gguf +``` diff --git a/tools/tts/tts.cpp b/tools/tts/tts.cpp index 8578f8869151..6354be6d6712 100644 --- a/tools/tts/tts.cpp +++ b/tools/tts/tts.cpp @@ -69,6 +69,11 @@ int main(int argc, char ** argv) { // always enable embd, so that we can pass hidden states to the audio generation helper params.embedding = true; + // for chatterbox multilingual: its cfg pairs slot i with sequence n_parallel + i in a unified kv cache + // TODO: this applies to every model, find a better way for a pipeline to request extra sequences + params.n_parallel *= 2; + params.kv_unified = true; + llama_backend_init(); llama_numa_init(params.numa); From 149f5641b931aaf33697d70f9abc90c3e5300247 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Tue, 3 Nov 2026 23:57:26 +0100 Subject: [PATCH 036/286] mtmd: accurate support for qwen3-vl video prompt format (#31619) --- tests/test-mtmd-impl.cpp | 160 +++++++++++++ tools/mtmd/mtmd-helper.cpp | 465 ++++++++++++++++++------------------- tools/mtmd/mtmd-helper.h | 63 +++++ tools/mtmd/mtmd-image.cpp | 28 ++- tools/mtmd/mtmd-image.h | 14 +- tools/mtmd/mtmd-internal.h | 7 + tools/mtmd/mtmd.cpp | 45 +++- tools/mtmd/mtmd.h | 1 + 8 files changed, 511 insertions(+), 272 deletions(-) diff --git a/tests/test-mtmd-impl.cpp b/tests/test-mtmd-impl.cpp index 536dfaa8087f..b837f45da286 100644 --- a/tests/test-mtmd-impl.cpp +++ b/tests/test-mtmd-impl.cpp @@ -1,11 +1,13 @@ #include "testing.h" #include "mtmd-audio.h" +#include "mtmd-helper.h" #include "mtmd-image.h" #include "mtmd-internal.h" #include #include +#include #include #include #include @@ -92,6 +94,33 @@ MAKE_TEST(test_image_preprocessor_dyn_size) { } } +MAKE_TEST(test_image_preprocessor_qwen3vl_video) { + clip_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { n_frames, frame size, expected size from Qwen3VLVideoProcessor smart_resize } + const std::vector> cases = { + { 20, { 720, 358 }, { 704, 352 } }, // 720 / 32 = 22.5 rounds half to even + { 768, { 1920, 1080 }, { 224, 128 } }, // over the longest_edge budget + { 4, { 16, 40 }, { 32, 64 } }, // edge below the factor is upscaled first + { 2, { 32, 32 }, { 64, 64 } }, // under the shortest_edge budget + { 5, { 2760, 1552 }, { 2752, 1536 } }, // 5 / 2 = 2.5 rounds half to even + { 4, { 250, 6000 }, { 256, 6016 } }, + }; + + for (const auto & [n_frames, input, expected] : cases) { + clip_image_size actual; + const std::string name = std::to_string(n_frames) + " frames of " + std::to_string(input.width) + "x" + std::to_string(input.height); + if (!t.assert_true(name + " has a frame size", preproc.get_video_frame_size(n_frames, input.width, input.height, actual))) { + continue; + } + t.assert_equal(name + " width", expected.width, actual.width); + t.assert_equal(name + " height", expected.height, actual.height); + } +} + static std::string image_size_to_str(const clip_image_size & size) { return std::to_string(size.width) + "x" + std::to_string(size.height); } @@ -301,6 +330,130 @@ MAKE_TEST(test_temporal_merge_grouping) { } } +// +// mtmd_helper video +// + +MAKE_TEST(test_video_config) { + clip_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { template, n_frames and fps of the source, expected fps, n_frames and frame size } + // the 1920x1080 sizes are from Qwen3VLVideoProcessor smart_resize + const std::vector> cases = { + { MTMD_VIDEO_TEMPLATE_QWEN3VL, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, + { MTMD_VIDEO_TEMPLATE_LFM3, 36000, 30.0f, 2.0f, 768, { 224, 128 } }, + { MTMD_VIDEO_TEMPLATE_LFM3_OMNI, 36000, 30.0f, 2.0f, 384, { 320, 192 } }, + { MTMD_VIDEO_TEMPLATE_QWEN3VL, -1, 30.0f, 2.0f, 0, { 0, 0 } }, // frame count unknown + { MTMD_VIDEO_TEMPLATE_DEFAULT, 36000, 30.0f, 0.0f, 0, { 0, 0 } }, + }; + + for (const auto & [tmpl, n_frames, fps, expected_fps, expected_n_frames, expected_size] : cases) { + const mtmd_video_config cfg = mtmd_video_config_init(tmpl, &preproc, 1920, 1080, n_frames, fps); + const std::string name = "template " + std::to_string(tmpl) + " with " + std::to_string(n_frames) + " frames"; + t.assert_equal(name + " template", (int) tmpl, (int) cfg.tmpl); + t.assert_equal(name + " fps", expected_fps, cfg.fps); + t.assert_equal(name + " n_frames", expected_n_frames, cfg.n_frames); + t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ cfg.frame_width, cfg.frame_height })); + } +} + +// n_frames source frames, each one starts with its index +struct test_video_source : mtmd_helper_video_seq { + int32_t n_frames; + int32_t n_read = 0; + + test_video_source(int32_t n_frames) : n_frames(n_frames) {} + + bool read_raw_frame() override { + if (n_read >= n_frames) { + return false; + } + frame_buf[0] = static_cast(n_read & 255); + frame_buf[1] = static_cast(n_read >> 8); + n_read++; + return true; + } +}; + +// text as is, a frame as [source index], an audio bitmap as {n_samples} +static std::string video_sequence_to_str(testing & t, mtmd_helper_video_seq & video) { + std::string out; + mtmd_bitmap * bitmap; + char * text; + int32_t ret; + while ((ret = video.read_next(&bitmap, &text)) == 0) { + if (text) { + out += text; + free(text); + } else if (mtmd_bitmap_is_audio(bitmap)) { + out += "{" + std::to_string(mtmd_bitmap_get_n_bytes(bitmap) / sizeof(float)) + "}"; + mtmd_bitmap_free(bitmap); + } else { + const unsigned char * data = mtmd_bitmap_get_data(bitmap); + out += "[" + std::to_string(data[0] | data[1] << 8) + "]"; + mtmd_bitmap_free(bitmap); + } + } + t.assert_equal("end of stream", -1, ret); + return out; +} + +MAKE_TEST(test_video_sequence_qwen3vl) { + clip_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_qwen3vl preproc(hparams); + + // { source video, expected frame size and sequence from Qwen3VLProcessor (transformers 4.57.1) } + const std::vector> cases = { + { { 720, 358, 24.0f, 240 }, { 704, 352 }, + "<0.3 seconds>[0][13]<1.3 seconds>[25][38]<2.4 seconds>[50][63]<3.4 seconds>[75][88]<4.5 seconds>[101][113]" + "<5.5 seconds>[126][138]<6.6 seconds>[151][164]<7.6 seconds>[176][189]<8.6 seconds>[201][214]<9.7 seconds>[226][239]" }, + // odd frame count: the last group repeats its last frame + { { 640, 480, 10.0f, 35 }, { 640, 480 }, + "<0.3 seconds>[0][6]<1.4 seconds>[11][17]<2.5 seconds>[23][28]<3.4 seconds>[34]" }, + // fewer frames than the minimum of 4 + { { 320, 240, 30.0f, 3 }, { 320, 256 }, + "<0.0 seconds>[0][1]<0.1 seconds>[2]" }, + { { 1280, 720, 30000.0f / 1001.0f, 300 }, { 1280, 704 }, + "<0.3 seconds>[0][16]<1.3 seconds>[31][47]<2.4 seconds>[63][79]<3.4 seconds>[94][110]<4.5 seconds>[126][142]" + "<5.5 seconds>[157][173]<6.6 seconds>[189][205]<7.6 seconds>[220][236]<8.7 seconds>[252][268]<9.7 seconds>[283][299]" }, + }; + + for (const auto & [src, expected_size, expected] : cases) { + const std::string name = std::to_string(src.n_frames) + " frames of " + std::to_string(src.width) + "x" + std::to_string(src.height); + const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_QWEN3VL, &preproc, src.width, src.height, src.n_frames, src.fps); + test_video_source video(src.n_frames); + video.configure(cfg, src, 0.0f); + video.src_alive = true; + t.assert_equal(name + " frame size", image_size_to_str(expected_size), image_size_to_str({ (int) video.info.width, (int) video.info.height })); + t.assert_equal(name + " sequence", expected, video_sequence_to_str(t, video)); + } +} + +// the soundtrack of each group comes after its frames +MAKE_TEST(test_video_sequence_lfm3_omni) { + clip_hparams hparams; + hparams.patch_size = 16; + hparams.n_merge = 2; + const mtmd_image_preprocessor_lfm3vl preproc(hparams); + + const mtmd_helper_video_src src = { 64, 64, 2.0f, 9 }; + const mtmd_video_config cfg = mtmd_video_config_init(MTMD_VIDEO_TEMPLATE_LFM3_OMNI, &preproc, src.width, src.height, src.n_frames, src.fps); + test_video_source video(src.n_frames); + video.configure(cfg, src, 0.0f); + video.src_alive = true; + video.audio_rate = 100; + video.soundtrack.assign(450, 0.0f); + + t.assert_equal("sequence", + std::string("<0.2 seconds>[0][1]{100}<1.2 seconds>[2][3]{100}<2.2 seconds>[4][5]{100}<3.2 seconds>[6][7]{100}<4.0 seconds>[8]{50}"), + video_sequence_to_str(t, video)); +} + // // mtmd_audio // @@ -357,6 +510,13 @@ int main(int argc, char ** argv) { testing t(std::cout); t.verbose = true; + // the video helper logs every frame at debug level + mtmd_helper_log_set([](ggml_log_level level, const char * text, void *) { + if (level != GGML_LOG_LEVEL_DEBUG) { + fputs(text, stderr); + } + }, nullptr); + // usage: test-mtmd-impl [filter_regex] for (int i = 1; i < argc; i++) { t.set_filter(argv[i]); diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index 29812219c2b2..d86537e55b54 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -508,8 +508,6 @@ mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(const mtmd_context // Video input helpers // -#ifdef MTMD_VIDEO - // MSVC deprecates strdup(), use a std::string version directly static char * mtmd_strdup(const std::string & s) { size_t n = s.size() + 1; @@ -520,15 +518,218 @@ static char * mtmd_strdup(const std::string & s) { return p; } -struct mtmd_helper_video { +mtmd_helper_video_seq::~mtmd_helper_video_seq() { + mtmd_bitmap_free(pending_frame); + mtmd_bitmap_free(pending_audio); +} + +int32_t mtmd_helper_video_seq::ts_per_n_frames() const { + return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI || tmpl == MTMD_VIDEO_TEMPLATE_QWEN3VL ? 2 : 0; +} + +void mtmd_helper_video_seq::configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg) { + tmpl = cfg.tmpl; + uint32_t width = src.width; + uint32_t height = src.height; + float duration = src.duration; + if (duration < 0.0f && src.n_frames > 0) { + duration = (float)src.n_frames / src.fps; + } + + // the model's recommended sampling rate wins over the generic default + fps_target = cfg.fps > 0.0f ? cfg.fps + : fps_target_arg > 0.0f ? fps_target_arg : src.fps; + + if (cfg.n_frames > 0 && duration > 0.0f) { + const int32_t n = cfg.n_frames; + target_frames = n; + fps_target = (float) n / duration; + + // same frames as the reference processors: round(linspace(0, n_frames - 1, n)) + const double step = n > 1 ? (double) (src.n_frames - 1) / (n - 1) : 0.0; + frame_indices.resize(n); + for (int32_t i = 0; i < n; i++) { + frame_indices[i] = (int32_t) std::nearbyint(i * step); + } + src_fps = src.fps; + n_src_frames = src.n_frames; + + if (cfg.frame_width > 0 && cfg.frame_height > 0 && + (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) { + out_width = cfg.frame_width; + out_height = cfg.frame_height; + } + } + + if (out_width > 0 && out_height > 0) { + width = (uint32_t) out_width; + height = (uint32_t) out_height; + } + + info.width = width; + info.height = height; + info.fps = fps_target; + LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n", + __func__, width, height, fps_target, duration, src.n_frames, target_frames); + info.n_frames = target_frames > 0 ? target_frames + : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; + frame_buf.resize((size_t)width * height * 3); +} + +// the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended +mtmd_bitmap * mtmd_helper_video_seq::soundtrack_slice(int32_t idx) const { + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + const int32_t group = idx / ts_per_n_frames; + const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames; + auto group_start = [&](int32_t g) { + const double t = frame_indices[g * ts_per_n_frames] / src_fps; + return std::min(soundtrack.size(), (size_t) std::nearbyint(t * audio_rate)); + }; + const size_t begin = group == 0 ? 0 : group_start(group); + const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1); + if (begin >= end) { + return nullptr; + } + mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin); + if (!id.empty()) { + mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str()); + } + return slice; +} + +mtmd_bitmap * mtmd_helper_video_seq::read_next_frame() { + if (!src_alive) return nullptr; + + if (!frame_indices.empty()) { + if (next_index >= frame_indices.size()) { + return nullptr; + } + // skip to the wanted source frame; a repeated index reuses the frame in frame_buf + const int32_t want = frame_indices[next_index]; + while (src_frame <= want) { + if (!read_raw_frame()) { + src_alive = false; + return nullptr; + } + src_frame++; + } + next_index++; + } else if (!read_raw_frame()) { + src_alive = false; + return nullptr; + } + + LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); + mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); + mtmd_bitmap_set_mergeable(frame, true); + mtmd_bitmap_set_is_video(frame, true); + if (!id.empty()) { + // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache + std::string frame_id = id + "+" + std::to_string(current_frame); + mtmd_bitmap_set_id(frame, frame_id.c_str()); + } + current_frame++; + return frame; +} + +int32_t mtmd_helper_video_seq::read_next(mtmd_bitmap ** out_bitmap, char ** out_text) { + *out_bitmap = nullptr; + *out_text = nullptr; + + if (!pending_text.empty()) { + *out_text = mtmd_strdup(pending_text); + pending_text.clear(); + return *out_text ? 0 : -2; + } + + if (pending_frame) { + *out_bitmap = pending_frame; + pending_frame = nullptr; + return 0; + } + + if (pending_audio) { + *out_bitmap = pending_audio; + pending_audio = nullptr; + return 0; + } + + LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n", + __func__, (int)src_alive, (int)start_emitted, current_frame); + + if (!src_alive) { + return (current_frame == 0) ? -2 : -1; + } + + if (!start_emitted) { + start_emitted = true; + // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize + if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) { + *out_text = mtmd_strdup(prompt_start); + return *out_text ? 0 : -2; + } + } + + mtmd_bitmap * frame = read_next_frame(); + if (!frame) { + return -1; + } + + const int32_t ts_per_n_frames = this->ts_per_n_frames(); + if (ts_per_n_frames > 0) { + // emit the group timestamp before the group's first frame; the value is the + // midpoint time of the group, matching HF processors that stamp each temporal group + const int32_t idx = current_frame - 1; // read_next_frame() already incremented + // the soundtrack of a group follows its last frame + const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size(); + if (!soundtrack.empty() && !frame_indices.empty() && group_ends) { + pending_audio = soundtrack_slice(idx); + } + if (idx % ts_per_n_frames == 0) { + char ts_buf[64]; + double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps; + if (!frame_indices.empty()) { + // stamp from the source frames; a short last group repeats its last frame + const size_t first = idx; + const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1); + ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0; + } + snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s); + pending_frame = frame; + *out_text = mtmd_strdup(ts_buf); + return *out_text ? 0 : -2; + } + *out_bitmap = frame; + return 0; + } + + *out_bitmap = frame; + + if (timestamp_interval_ms > 0) { + // current_frame was already incremented by read_next_frame(); undo for elapsed calc + float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f; + if (elapsed_ms >= next_timestamp_ms) { + char ts_buf[32]; + float elapsed_s = elapsed_ms / 1000.0f; + int minutes = (int)(elapsed_s / 60); + float seconds = elapsed_s - minutes * 60.0f; + snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds); + pending_text = ts_buf; + next_timestamp_ms += (float)timestamp_interval_ms; + } + } + + return 0; +} + +#ifdef MTMD_VIDEO + +struct mtmd_helper_video : mtmd_helper_video_seq { const mtmd_context * mctx; std::string path; std::vector input_buf; // non-empty when initialized from buffer std::string ffmpeg_bin; std::string ffprobe_bin; - float fps_target = 0.0f; - mtmd_helper_video_info info = {}; - std::string id; // hash of the input video // RAII wrapper for managing subprocess struct subprocess_handle { @@ -596,46 +797,15 @@ struct mtmd_helper_video { }; subprocess_handle sp; - int32_t current_frame = 0; - - std::string prompt_start = "Video:"; - int32_t timestamp_interval_ms = 5000; // emit a timestamp text every N ms (0 = disabled) - float next_timestamp_ms = 0.0f; // next elapsed-ms threshold at which to emit // apply mtmd_get_video_config(); false for still images decoded via ffmpeg - bool use_video_config = false; - mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; - - int32_t target_frames = 0; // exact frame count to emit (0 = no limit) - int32_t out_width = 0; // frame size from mtmd_get_video_config (0 = source size) - int32_t out_height = 0; - - // source frames to emit, as picked by the reference processors (empty = use the fps filter) - std::vector frame_indices; - size_t next_index = 0; - int32_t src_frame = 0; // source frames read from ffmpeg so far - double src_fps = 0.0; - - std::vector frame_buf; - std::string pending_text; // text queued to be returned before the next frame - mtmd_bitmap * pending_frame = nullptr; // frame queued to be returned after its group timestamp - - // soundtrack: after each group of frames comes the audio from its first frame to the next group's - double video_start_s = 0.0; - int32_t n_src_frames = 0; - std::vector soundtrack; - mtmd_bitmap * pending_audio = nullptr; - bool start_emitted = false; + bool use_video_config = false; + double video_start_s = 0.0; // start time of the video stream, to align the soundtrack bool is_buf_input() const { return !input_buf.empty(); } - // number of frames per timestamp group (0 = timestamp every timestamp_interval_ms) - int32_t ts_per_n_frames() const { - return tmpl == MTMD_VIDEO_TEMPLATE_LFM3 || tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 2 : 0; - } - bool probe(float fps_target_arg) { // cache: makes stdin seekable; without it ffprobe reports width/height 0 // for mp4 files whose moov atom sits at the end @@ -671,11 +841,7 @@ struct mtmd_helper_video { probe_sp.start_feeder(input_buf); } - uint32_t width = 0; - uint32_t height = 0; - float orig_fps = 0.0f; - float duration = -1.0f; - int32_t n_frames_orig = -1; + mtmd_helper_video_src src; char line[256]; FILE * fp = probe_sp.stdout_pipe(); @@ -689,15 +855,15 @@ struct mtmd_helper_video { if (nl) *nl = '\0'; if (strcmp(key, "width") == 0) { - width = (uint32_t)atoi(val); + src.width = (uint32_t)atoi(val); } else if (strcmp(key, "height") == 0) { - height = (uint32_t)atoi(val); + src.height = (uint32_t)atoi(val); } else if (strcmp(key, "r_frame_rate") == 0) { - orig_fps = parse_rational(val); + src.fps = parse_rational(val); } else if (strcmp(key, "nb_frames") == 0 && strcmp(val, "N/A") != 0) { - n_frames_orig = atoi(val); + src.n_frames = atoi(val); } else if (strcmp(key, "duration") == 0 && strcmp(val, "N/A") != 0) { - duration = (float)atof(val); + src.duration = (float)atof(val); } else if (strcmp(key, "start_time") == 0 && strcmp(val, "N/A") != 0) { video_start_s = atof(val); } @@ -705,58 +871,15 @@ struct mtmd_helper_video { probe_sp.stop(); - if (width == 0 || height == 0 || orig_fps <= 0.0f) { + if (src.width == 0 || src.height == 0 || src.fps <= 0.0f) { return false; } - if (duration < 0.0f && n_frames_orig > 0) { - duration = (float)n_frames_orig / orig_fps; - } - mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; if (use_video_config) { - cfg = mtmd_get_video_config(mctx, width, height, n_frames_orig, orig_fps); - tmpl = cfg.tmpl; - } - - // the model's recommended sampling rate wins over the generic default - fps_target = cfg.fps > 0.0f ? cfg.fps - : fps_target_arg > 0.0f ? fps_target_arg : orig_fps; - - if (cfg.n_frames > 0 && duration > 0.0f) { - const int32_t n = cfg.n_frames; - target_frames = n; - fps_target = (float) n / duration; - - // same frames as the reference processors: round(linspace(0, n_frames - 1, n)) - const double step = n > 1 ? (double) (n_frames_orig - 1) / (n - 1) : 0.0; - frame_indices.resize(n); - for (int32_t i = 0; i < n; i++) { - frame_indices[i] = (int32_t) std::nearbyint(i * step); - } - src_fps = orig_fps; - n_src_frames = n_frames_orig; - - if (cfg.frame_width > 0 && cfg.frame_height > 0 && - (cfg.frame_width != (int32_t) width || cfg.frame_height != (int32_t) height)) { - out_width = cfg.frame_width; - out_height = cfg.frame_height; - } + cfg = mtmd_get_video_config(mctx, src.width, src.height, src.n_frames, src.fps); } - - if (out_width > 0 && out_height > 0) { - width = (uint32_t) out_width; - height = (uint32_t) out_height; - } - - info.width = width; - info.height = height; - info.fps = fps_target; - LOG_DBG("%s: %ux%u fps=%.2f duration=%.2fs n_frames=%d target_frames=%d\n", - __func__, width, height, fps_target, duration, n_frames_orig, target_frames); - info.n_frames = target_frames > 0 ? target_frames - : duration > 0.0f ? (int32_t)(duration * fps_target + 0.5f) : -1; - frame_buf.resize((size_t)width * height * 3); + configure(cfg, src, fps_target_arg); return true; } @@ -800,31 +923,10 @@ struct mtmd_helper_video { LOG_WRN("%s: the video has no soundtrack\n", __func__); return; } + audio_rate = rate; soundtrack.assign(pcm.begin() + start, pcm.begin() + std::min(pcm.size(), start + len)); } - // the soundtrack of the group of frames that ends with frame idx, nullptr if the soundtrack has ended - mtmd_bitmap * soundtrack_slice(int32_t idx) const { - const int32_t ts_per_n_frames = this->ts_per_n_frames(); - const int32_t group = idx / ts_per_n_frames; - const int32_t n_groups = ((int32_t) frame_indices.size() + ts_per_n_frames - 1) / ts_per_n_frames; - const int rate = mtmd_get_audio_sample_rate(mctx); - auto group_start = [&](int32_t g) { - const double t = frame_indices[g * ts_per_n_frames] / src_fps; - return std::min(soundtrack.size(), (size_t) std::nearbyint(t * rate)); - }; - const size_t begin = group == 0 ? 0 : group_start(group); - const size_t end = group == n_groups - 1 ? soundtrack.size() : group_start(group + 1); - if (begin >= end) { - return nullptr; - } - mtmd_bitmap * slice = mtmd_bitmap_init_from_audio(end - begin, soundtrack.data() + begin); - if (!id.empty()) { - mtmd_bitmap_set_id(slice, (id + "+audio" + std::to_string(group)).c_str()); - } - return slice; - } - bool start_ffmpeg(float seek_seconds) { char seek_buf[64]; char fps_buf[64]; @@ -917,6 +1019,7 @@ struct mtmd_helper_video { sp.start_feeder(input_buf); } + src_alive = sp.alive; return sp.alive; } @@ -925,7 +1028,7 @@ struct mtmd_helper_video { } // read one raw frame from ffmpeg into frame_buf - bool read_raw_frame() { + bool read_raw_frame() override { FILE * fp = sp.stdout_pipe(); const size_t frame_size = (size_t)info.width * info.height * 3; LOG_DBG("%s: reading frame %d, expecting %zu bytes (%ux%u)\n", @@ -938,7 +1041,6 @@ struct mtmd_helper_video { // clean EOF only if no bytes read yet; partial frame is an error LOG_DBG("%s: fread returned 0 after %zu/%zu bytes (ferror=%d)\n", __func__, total_read, frame_size, ferror(fp)); - sp.alive = false; return false; } total_read += n; @@ -946,129 +1048,6 @@ struct mtmd_helper_video { return true; } - mtmd_bitmap * read_next_frame() { - if (!sp.alive) return nullptr; - - if (!frame_indices.empty()) { - if (next_index >= frame_indices.size()) { - return nullptr; - } - // skip to the wanted source frame; a repeated index reuses the frame in frame_buf - const int32_t want = frame_indices[next_index]; - while (src_frame <= want) { - if (!read_raw_frame()) { - return nullptr; - } - src_frame++; - } - next_index++; - } else if (!read_raw_frame()) { - return nullptr; - } - - LOG_DBG("%s: frame %d read OK\n", __func__, current_frame); - mtmd_bitmap * frame = mtmd_bitmap_init(info.width, info.height, frame_buf.data()); - mtmd_bitmap_set_mergeable(frame, true); - mtmd_bitmap_set_is_video(frame, true); - if (!id.empty()) { - // each frame gets a unique id in the form of {hash}+{frame}, so that it can be identified in cache - std::string frame_id = id + "+" + std::to_string(current_frame); - mtmd_bitmap_set_id(frame, frame_id.c_str()); - } - current_frame++; - return frame; - } - - int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text) { - *out_bitmap = nullptr; - *out_text = nullptr; - - if (!pending_text.empty()) { - *out_text = mtmd_strdup(pending_text); - pending_text.clear(); - return *out_text ? 0 : -2; - } - - if (pending_frame) { - *out_bitmap = pending_frame; - pending_frame = nullptr; - return 0; - } - - if (pending_audio) { - *out_bitmap = pending_audio; - pending_audio = nullptr; - return 0; - } - - LOG_DBG("%s: proc_alive=%d start_emitted=%d current_frame=%d\n", - __func__, (int)sp.alive, (int)start_emitted, current_frame); - - if (!sp.alive) { - return (current_frame == 0) ? -2 : -1; - } - - if (!start_emitted) { - start_emitted = true; - // model templates have no prompt prefix, the video begin/end tokens are added by mtmd_tokenize - if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT && !prompt_start.empty()) { - *out_text = mtmd_strdup(prompt_start); - return *out_text ? 0 : -2; - } - } - - mtmd_bitmap * frame = read_next_frame(); - if (!frame) { - return -1; - } - - const int32_t ts_per_n_frames = this->ts_per_n_frames(); - if (ts_per_n_frames > 0) { - // emit the group timestamp before the group's first frame; the value is the - // midpoint time of the group, matching HF processors that stamp each temporal group - const int32_t idx = current_frame - 1; // read_next_frame() already incremented - // the soundtrack of a group follows its last frame - const bool group_ends = idx % ts_per_n_frames == ts_per_n_frames - 1 || idx + 1 == (int32_t) frame_indices.size(); - if (!soundtrack.empty() && !frame_indices.empty() && group_ends) { - pending_audio = soundtrack_slice(idx); - } - if (idx % ts_per_n_frames == 0) { - char ts_buf[64]; - double ts_s = ((double)idx + 0.5*(double)(ts_per_n_frames - 1)) / info.fps; - if (!frame_indices.empty()) { - // stamp from the source frames; a short last group repeats its last frame - const size_t first = idx; - const size_t last = std::min(first + ts_per_n_frames - 1, frame_indices.size() - 1); - ts_s = (frame_indices[first] / src_fps + frame_indices[last] / src_fps) / 2.0; - } - snprintf(ts_buf, sizeof(ts_buf), "<%.1f seconds>", ts_s); - pending_frame = frame; - *out_text = mtmd_strdup(ts_buf); - return *out_text ? 0 : -2; - } - *out_bitmap = frame; - return 0; - } - - *out_bitmap = frame; - - if (timestamp_interval_ms > 0) { - // current_frame was already incremented by read_next_frame(); undo for elapsed calc - float elapsed_ms = (float)(current_frame - 1) / info.fps * 1000.0f; - if (elapsed_ms >= next_timestamp_ms) { - char ts_buf[32]; - float elapsed_s = elapsed_ms / 1000.0f; - int minutes = (int)(elapsed_s / 60); - float seconds = elapsed_s - minutes * 60.0f; - snprintf(ts_buf, sizeof(ts_buf), "[%dm%.2fs]", minutes, seconds); - pending_text = ts_buf; - next_timestamp_ms += (float)timestamp_interval_ms; - } - } - - return 0; - } - static float parse_rational(const char * s) { int num = 0, den = 1; if (sscanf(s, "%d/%d", &num, &den) == 2 && den > 0) { @@ -1233,8 +1212,6 @@ void mtmd_helper_video_free(mtmd_helper_video * ctx) { #ifdef MTMD_VIDEO if (!ctx) return; ctx->stop_ffmpeg(); - mtmd_bitmap_free(ctx->pending_frame); - mtmd_bitmap_free(ctx->pending_audio); delete ctx; #else GGML_UNUSED(ctx); diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index e562aae1823c..a9ffdf71e6b4 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -318,6 +318,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); #ifdef __cplusplus #include #include +#include #include namespace mtmd_helper { @@ -408,6 +409,68 @@ inline std::vector audio_stream_logit_bias(const mtmd_context } } // namespace mtmd_helper + +// +// test API (C++ only) +// internals of mtmd_helper_video, do not use in applications +// + +struct mtmd_helper_video_src { + uint32_t width = 0; + uint32_t height = 0; + float fps = 0.0f; + int32_t n_frames = -1; + float duration = -1.0f; +}; + +struct mtmd_helper_video_seq { + float fps_target = 0.0f; + mtmd_helper_video_info info = {}; + std::string id; + + bool src_alive = false; + int32_t current_frame = 0; + + std::string prompt_start = "Video:"; + int32_t timestamp_interval_ms = 5000; + float next_timestamp_ms = 0.0f; + + mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; + + int32_t target_frames = 0; + int32_t out_width = 0; + int32_t out_height = 0; + + std::vector frame_indices; + size_t next_index = 0; + int32_t src_frame = 0; + double src_fps = 0.0; + + std::vector frame_buf; + std::string pending_text; + mtmd_bitmap * pending_frame = nullptr; + + int32_t n_src_frames = 0; + int audio_rate = 0; + std::vector soundtrack; + mtmd_bitmap * pending_audio = nullptr; + bool start_emitted = false; + + mtmd_helper_video_seq() = default; + mtmd_helper_video_seq(const mtmd_helper_video_seq &) = delete; + mtmd_helper_video_seq & operator=(const mtmd_helper_video_seq &) = delete; + virtual ~mtmd_helper_video_seq(); + + virtual bool read_raw_frame() = 0; + + void configure(const mtmd_video_config & cfg, const mtmd_helper_video_src & src, float fps_target_arg); + + int32_t ts_per_n_frames() const; + + mtmd_bitmap * soundtrack_slice(int32_t idx) const; + mtmd_bitmap * read_next_frame(); + int32_t read_next(mtmd_bitmap ** out_bitmap, char ** out_text); +}; #endif #endif diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 2750bc3fc61b..8c0f433e39b1 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -819,12 +819,12 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i } // -// mtmd_image_preprocessor_lfm3vl +// mtmd_image_preprocessor_qwen3vl // // rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does -clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, - int n_frames, int temporal_factor) { +clip_image_size mtmd_image_preprocessor_qwen3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels, + int n_frames, int temporal_factor) { double h = height; double w = width; // the video variant first upscales edges below factor @@ -850,18 +850,13 @@ clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int heig return {std::max(factor, w_bar), std::max(factor, h_bar)}; } -clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const { - return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge, - hparams.image_min_pixels, hparams.image_max_pixels); -} - -bool mtmd_image_preprocessor_lfm3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const { - // Lfm3VlVideoProcessor size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 +bool mtmd_image_preprocessor_qwen3vl::get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const { + // Qwen3VLVideoProcessor (and Lfm3VlVideoProcessor) size: shortest_edge 4096, longest_edge 25165824, temporal_patch_size 2 out = smart_resize(width, height, hparams.patch_size * hparams.n_merge, 4096, 25165824, n_frames, 2); return true; } -mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(const clip_image_u8 & img) const { +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::preprocess_video_frame(const clip_image_u8 & img) const { const int f = hparams.patch_size * hparams.n_merge; const auto size = img.get_size(); return resize_to(img, { @@ -870,7 +865,7 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::preprocess_video_frame(co }); } -mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const { +mtmd_image_preproc_out mtmd_image_preprocessor_qwen3vl::resize_to(const clip_image_u8 & img, const clip_image_size & size) const { clip_image_u8 resized_image; img_tool::resize(img, resized_image, size, hparams.image_resize_algo, @@ -881,6 +876,15 @@ mtmd_image_preproc_out mtmd_image_preprocessor_lfm3vl::resize_to(const clip_imag return output; } +// +// mtmd_image_preprocessor_lfm3vl +// + +clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const { + return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge, + hparams.image_min_pixels, hparams.image_max_pixels); +} + // // mtmd_image_preprocessor_glm5v // diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 07feae4b2305..6eb49ca34706 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -137,15 +137,15 @@ struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor { // this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor { mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} + mtmd_image_preprocessor_dyn_size(const clip_hparams & hparams) : mtmd_image_preprocessor(hparams) {} static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size); mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override; virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); } }; -// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize -struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size { - mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {} - clip_image_size target_size(const clip_image_size & original_size) const override; +// Qwen3-VL, same as dyn_size for images, video frames are sized as Qwen3VLVideoProcessor +struct mtmd_image_preprocessor_qwen3vl : mtmd_image_preprocessor_dyn_size { + using mtmd_image_preprocessor_dyn_size::mtmd_image_preprocessor_dyn_size; // the video helper sizes the frames (get_video_frame_size), they are only snapped to the grid mtmd_image_preproc_out preprocess_video_frame(const clip_image_u8 & img) const override; bool get_video_frame_size(int n_frames, int width, int height, clip_image_size & out) const override; @@ -156,6 +156,12 @@ struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size { int n_frames = 1, int temporal_factor = 1); }; +// LFM3-VL, same as qwen3vl but images are sized exactly as the transformers Qwen2-VL smart_resize +struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_qwen3vl { + using mtmd_image_preprocessor_qwen3vl::mtmd_image_preprocessor_qwen3vl; + clip_image_size target_size(const clip_image_size & original_size) const override; +}; + // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally struct mtmd_image_preprocessor_glm5v : mtmd_image_preprocessor { mtmd_image_preprocessor_glm5v(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {} diff --git a/tools/mtmd/mtmd-internal.h b/tools/mtmd/mtmd-internal.h index e7c62773e38f..eb961d31edca 100644 --- a/tools/mtmd/mtmd-internal.h +++ b/tools/mtmd/mtmd-internal.h @@ -19,3 +19,10 @@ struct mtmd_internal_part { // [QWEN_VIDEO] merged parts are erased from `parts`, so one group always maps to one part std::vector> mtmd_group_mergeable_bitmaps(std::vector & parts, int n_merge); + +struct mtmd_image_preprocessor; + +// mtmd_get_video_config() for a given template, without a context +mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps); diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 1c89eabfa355..128e0ebd6c5f 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -805,9 +805,15 @@ struct mtmd_context { image_preproc = std::make_unique(ctx_v); } } break; + case PROJECTOR_TYPE_QWEN3VL: + { + // <|vision_start|> ... (image embeddings) ... <|vision_end|> + img_beg = "<|vision_start|>"; + img_end = "<|vision_end|>"; + image_preproc = std::make_unique(ctx_v); + } break; case PROJECTOR_TYPE_QWEN2VL: case PROJECTOR_TYPE_QWEN25VL: - case PROJECTOR_TYPE_QWEN3VL: case PROJECTOR_TYPE_LING3VL: case PROJECTOR_TYPE_MIMOVL: { @@ -3259,36 +3265,51 @@ int mtmd_get_audio_sample_rate(const mtmd_context * ctx) { return clip_get_hparams(ctx->ctx_a)->audio_sample_rate; } -mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, - int32_t src_width, int32_t src_height, - int32_t src_n_frames, float src_fps) { - mtmd_video_config cfg{MTMD_VIDEO_TEMPLATE_DEFAULT, 0.0f, 0, 0, 0}; - if (!ctx->ctx_v || clip_get_projector_type(ctx->ctx_v) != PROJECTOR_TYPE_LFM3VL) { +mtmd_video_config mtmd_video_config_init(mtmd_video_template tmpl, const mtmd_image_preprocessor * preproc, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps) { + mtmd_video_config cfg{tmpl, 0.0f, 0, 0, 0}; + if (tmpl == MTMD_VIDEO_TEMPLATE_DEFAULT) { return cfg; } - // matches Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp + // matches Qwen3VLVideoProcessor and Lfm3VlVideoProcessor: 2 fps sampling with a [4, 768] frame clamp // LFM3-Omni samples at most 384 frames and hears the soundtrack - const bool omni = ctx->video_has_soundtrack(); - cfg.tmpl = omni ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; - cfg.fps = 2.0f; + cfg.fps = 2.0f; if (src_n_frames <= 0 || src_fps <= 0.0f) { return cfg; } int32_t n = (int32_t)((double) src_n_frames / src_fps * cfg.fps); - n = std::clamp(n, 4, omni ? 384 : 768); + n = std::clamp(n, 4, tmpl == MTMD_VIDEO_TEMPLATE_LFM3_OMNI ? 384 : 768); n = std::min(n, src_n_frames); cfg.n_frames = n; clip_image_size size; - if (ctx->image_preproc && ctx->image_preproc->get_video_frame_size(n, src_width, src_height, size)) { + if (preproc && preproc->get_video_frame_size(n, src_width, src_height, size)) { cfg.frame_width = size.width; cfg.frame_height = size.height; } return cfg; } +mtmd_video_config mtmd_get_video_config(const mtmd_context * ctx, + int32_t src_width, int32_t src_height, + int32_t src_n_frames, float src_fps) { + mtmd_video_template tmpl = MTMD_VIDEO_TEMPLATE_DEFAULT; + switch (ctx->ctx_v ? clip_get_projector_type(ctx->ctx_v) : PROJECTOR_TYPE_UNKNOWN) { + case PROJECTOR_TYPE_LFM3VL: + tmpl = ctx->video_has_soundtrack() ? MTMD_VIDEO_TEMPLATE_LFM3_OMNI : MTMD_VIDEO_TEMPLATE_LFM3; + break; + case PROJECTOR_TYPE_QWEN3VL: + tmpl = MTMD_VIDEO_TEMPLATE_QWEN3VL; + break; + default: + break; + } + return mtmd_video_config_init(tmpl, ctx->image_preproc.get(), src_width, src_height, src_n_frames, src_fps); +} + const char * mtmd_get_marker(const mtmd_context * ctx) { return ctx->media_marker.c_str(); } diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index d490e9668b47..a593e7889607 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -63,6 +63,7 @@ enum mtmd_video_template { MTMD_VIDEO_TEMPLATE_DEFAULT, // "Video:" prefix, then a "[XmY.YYs]" timestamp every N ms MTMD_VIDEO_TEMPLATE_LFM3, // "" before every group of 2 frames MTMD_VIDEO_TEMPLATE_LFM3_OMNI, // same as LFM3, then the soundtrack of each group, as an audio bitmap after its frames + MTMD_VIDEO_TEMPLATE_QWEN3VL, // "" before every group of 2 frames, no markers around the whole video }; // opaque types From bee23322f1baa3d2dd16b8339bed6aff4a8eb663 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Tue, 3 Nov 2026 21:17:30 -0800 Subject: [PATCH 037/286] hexagon: add LIGHTNING_INDEXER support (#31289) * hexagon: initial support for LIGHTNING_INDEXER op * hex-lght-idxr: hmx for prefill * hex-lght-idxr: q8_0 decode gather transpose, two-block decode compute * hex-lght-idxr: pipelined hmx path for decode and prefill * hex-lid: add support for K (f32) * hex-lid: add support for K (f32) * hex-lid: add support for q4_0 * hex-lid: remove scalar VTCM reads in the HMX q4_0 K prep and remove vector spills * hex-lid: host/kernel kparams alighnment and remove sw devices in kernel * hex-lit: update dma pipeline to use canonical dbl buffering in all paths * hex-lit: add GGML_HEXAGON_LI_SELECT to force hmx/hvx paths * hex-lit: add support for dims collapse * hex-lit: make kparams constistent with other ops --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 3 + ggml/src/ggml-hexagon/htp-common.h | 1 + ggml/src/ggml-hexagon/htp-kparams.cpp | 258 +++ ggml/src/ggml-hexagon/htp-kparams.h | 1 + ggml/src/ggml-hexagon/htp-opnode.cpp | 7 + ggml/src/ggml-hexagon/htp/CMakeLists.txt | 1 + ggml/src/ggml-hexagon/htp/htp-ctx.h | 1 + ggml/src/ggml-hexagon/htp/htp-ops.h | 1 + .../ggml-hexagon/htp/lightning-indexer-ops.c | 1722 +++++++++++++++++ .../ggml-hexagon/htp/lightning-indexer-ops.h | 213 ++ ggml/src/ggml-hexagon/htp/main.c | 3 + scripts/snapdragon/run.py | 3 + tests/test-backend-ops.cpp | 3 + 13 files changed, 2217 insertions(+) create mode 100644 ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c create mode 100644 ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 6cbc475a0e1d..47e90b030e2d 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -135,6 +135,7 @@ size_t opt_mbuf_total = 0; // total buffer space limit ( int opt_mm_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) int opt_fa_select = 2; // 2 = HMX -> HVX -> CPU, 1 = HVX -> CPU, 0 = CPU (unsupported) int opt_fa_split = 1; // 1 = head-split + team-split when n_kv_heads < N_cores (default), 0 = off +int opt_li_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) int opt_gdn_select = 2; // 2 = HMX -> HVX, 1 = HVX, 0 = CPU (unsupported) int opt_ar_select = 2; // 2 = fused ALLREDUCE+ADD (default), 1 = unfused ALLREDUCE, 0 = fallback to CPY+FENCE int opt_ar_scatter = 1; // 1 = reduce-scatter the fused ALLREDUCE+ADD (default), 0 = full reduction @@ -3409,6 +3410,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { const char * str_mm_select = getenv("GGML_HEXAGON_MM_SELECT"); const char * str_fa_select = getenv("GGML_HEXAGON_FA_SELECT"); const char * str_fa_split = getenv("GGML_HEXAGON_FA_SPLIT"); + const char * str_li_select = getenv("GGML_HEXAGON_LI_SELECT"); const char * str_gdn_select = getenv("GGML_HEXAGON_GDN_SELECT"); const char * str_ar_select = getenv("GGML_HEXAGON_AR_SELECT"); const char * str_ar_scatter = getenv("GGML_HEXAGON_AR_SCATTER"); @@ -3475,6 +3477,7 @@ static void ggml_hexagon_init(ggml_backend_reg * reg) { opt_mm_select = str_mm_select ? atoi(str_mm_select) : opt_mm_select; opt_fa_select = str_fa_select ? atoi(str_fa_select) : opt_fa_select; opt_fa_split = str_fa_split ? atoi(str_fa_split) : opt_fa_split; + opt_li_select = str_li_select ? atoi(str_li_select) : opt_li_select; opt_gdn_select = str_gdn_select ? atoi(str_gdn_select) : opt_gdn_select; opt_ar_select = str_ar_select ? atoi(str_ar_select) : opt_ar_select; opt_ar_scatter = str_ar_scatter ? atoi(str_ar_scatter) : opt_ar_scatter; diff --git a/ggml/src/ggml-hexagon/htp-common.h b/ggml/src/ggml-hexagon/htp-common.h index e51340d8a278..472892be9ea5 100644 --- a/ggml/src/ggml-hexagon/htp-common.h +++ b/ggml/src/ggml-hexagon/htp-common.h @@ -29,6 +29,7 @@ extern size_t opt_mbuf_total; extern int opt_mm_select; extern int opt_fa_select; extern int opt_fa_split; +extern int opt_li_select; extern int opt_gdn_select; extern int opt_ar_select; extern int opt_ar_scatter; diff --git a/ggml/src/ggml-hexagon/htp-kparams.cpp b/ggml/src/ggml-hexagon/htp-kparams.cpp index 7362bb1c9f65..48bd932776d9 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.cpp +++ b/ggml/src/ggml-hexagon/htp-kparams.cpp @@ -64,6 +64,7 @@ htp_op_code htp_op_remap(const struct ggml_tensor * t) { case GGML_OP_CONV_2D_DW: return HTP_OP_CONV_2D_DW; case GGML_OP_ROLL: return HTP_OP_ROLL; case GGML_OP_UPSCALE: return HTP_OP_UPSCALE; + case GGML_OP_LIGHTNING_INDEXER: return HTP_OP_LIGHTNING_INDEXER; case GGML_OP_UNARY: switch (ggml_get_unary_op(t)) { @@ -4708,6 +4709,233 @@ static inline bool htp_kparams_solve_dsv4_hc_post( return htp_kparams_solve_dsv4_hc_post_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); } +static bool htp_lightning_indexer_can_collapse( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst +) { + if (!q || !k || !w || !m || !dst) { + return false; + } + + // 2D K normalization: ne[1] is n_kv, ne[2] is 1 + const bool can_collapse_k = (k->ne[1] > 1 && k->ne[2] == 1); + + // Stream-to-token collapse across q, w, dst (and m) + bool can_collapse_stream = false; + if (q->ne[3] > 1 && k->ne[3] == 1) { + const bool q_ok = (q->nb[3] == (size_t) q->ne[2] * q->nb[2]); + const bool dst_ok = (dst->ne[3] == q->ne[3] && dst->ne[2] == 1 && dst->ne[1] == q->ne[2] && + dst->nb[3] == (size_t) dst->ne[1] * dst->nb[1]); + const bool w_ok = (w->ne[3] == q->ne[3] && w->ne[2] == 1 && w->ne[1] == q->ne[2] && + w->nb[3] == (size_t) w->ne[1] * w->nb[1]) || + (w->ne[3] == 1 && w->ne[1] == q->ne[2] * q->ne[3]); + const bool m_ok = (m->ne[3] == q->ne[3] && m->ne[2] == 1 && m->ne[1] == q->ne[2] && + m->nb[3] == (size_t) m->ne[1] * m->nb[1]) || + (m->ne[3] == 1 && m->ne[1] == q->ne[2] * q->ne[3]); + can_collapse_stream = q_ok && dst_ok && w_ok && m_ok; + } + + return can_collapse_k || can_collapse_stream; +} + +static void htp_lightning_indexer_collapse_tensors( + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * dst, + struct ggml_tensor * q_c, + struct ggml_tensor * k_c, + struct ggml_tensor * w_c, + struct ggml_tensor * m_c, + struct ggml_tensor * dst_c +) { + *q_c = *q; + *k_c = *k; + *w_c = *w; + *m_c = *m; + *dst_c = *dst; + + if (k->ne[1] > 1 && k->ne[2] == 1) { + k_c->ne[2] = k->ne[1]; + k_c->ne[1] = 1; + k_c->nb[1] = k->nb[1]; + k_c->nb[2] = k->nb[1]; + k_c->nb[3] = k->nb[2]; + } + + if (q->ne[3] > 1 && k->ne[3] == 1) { + const int64_t n_tokens_collapsed = (int64_t) q->ne[2] * q->ne[3]; + + q_c->ne[2] = n_tokens_collapsed; + q_c->ne[3] = 1; + q_c->nb[3] = (size_t) q_c->ne[2] * q_c->nb[2]; + + if (w->ne[3] == q->ne[3]) { + w_c->ne[1] = n_tokens_collapsed; + w_c->ne[3] = 1; + w_c->nb[2] = (size_t) w_c->ne[1] * w_c->nb[1]; + w_c->nb[3] = w_c->nb[2]; + } + + dst_c->ne[1] = n_tokens_collapsed; + dst_c->ne[3] = 1; + dst_c->nb[2] = (size_t) dst_c->ne[1] * dst_c->nb[1]; + dst_c->nb[3] = dst_c->nb[2]; + + if (m->ne[3] == q->ne[3] && m->ne[1] == q->ne[2]) { + m_c->ne[1] = n_tokens_collapsed; + m_c->ne[3] = 1; + m_c->nb[2] = (size_t) m_c->ne[1] * m_c->nb[1]; + m_c->nb[3] = m_c->nb[2]; + } + } +} + +static bool htp_kparams_solve_lightning_indexer_impl( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * q, + const struct ggml_tensor * k, + const struct ggml_tensor * w, + const struct ggml_tensor * m, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (opt_li_select < 1) { + return false; + } + + if (k->type != GGML_TYPE_F32 && k->type != GGML_TYPE_F16 && k->type != GGML_TYPE_Q8_0 && k->type != GGML_TYPE_Q4_0) { + return false; + } + + if (q->ne[0] % 32 != 0 || + !ggml_is_contiguous_rows(q) || !ggml_is_contiguous_rows(k) || + !ggml_is_contiguous_rows(w) || !ggml_is_contiguous_rows(m) || + !ggml_is_contiguous_rows(op)) { + return false; + } + + const uint32_t n_kv_blocks = (k->ne[2] + HTP_LID_KV_BLOCK - 1) / HTP_LID_KV_BLOCK; + if (n_kv_blocks == 0 || q->ne[2] == 0) { + return false; + } + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_tokens = q->ne[2]; + const size_t k_row_size = ggml_row_size(k->type, D); + + kparams->k_row_size = (uint32_t) k_row_size; + kparams->div_n_head = init_fastdiv_values(n_head); + kparams->div_m_ne3 = init_fastdiv_values((uint32_t) m->ne[3]); + + if (opt_li_select >= 2 && htp_session_n_hmx(sess) > 0 && D % HMX_FP16_TILE_N_COLS == 0) { + const uint32_t n_threads = htp_session_n_threads(sess); + const uint32_t n_stream = q->ne[3]; + const uint32_t n_dot = D / HMX_FP16_TILE_N_COLS; + + auto vtcm_usage = [&](uint32_t n_tok, uint32_t kv_block, uint32_t n_qsets) { + return htp_lid_hmx_compute_vtcm_usage(D, n_head, n_tok, kv_block, n_qsets, n_threads, k_row_size); + }; + + uint32_t n_tok = (std::min)(n_tokens, (uint32_t) HTP_LID_HMX_TOKENS_MAX); + while (n_tok > 32 && vtcm_usage(n_tok, 64, 1) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + uint32_t n_qsets = n_tok == n_tokens ? (std::min)(n_stream, (uint32_t) HTP_LID_HMX_MAX_QSETS) : 1; + while (n_qsets > 1 && vtcm_usage(n_tok, 64, n_qsets) > htp_session_vtcm_size(sess)) { + n_qsets /= 2; + } + + uint32_t n_qbuf = 1; + if (n_qsets == 1 && n_head >= 16 && n_stream * ((n_tokens + n_tok - 1) / n_tok) > 1) { + uint32_t nt2 = n_tok; + while (nt2 > 32 && vtcm_usage(nt2, 64, 2) > htp_session_vtcm_size(sess)) { + nt2 /= 2; + } + const size_t q_bytes = (size_t) nt2 * n_head * D * sizeof(float); + const size_t k_bytes = (size_t) k->ne[2] * ggml_row_size(k->type, D); + if (vtcm_usage(nt2, 64, 2) <= htp_session_vtcm_size(sess) && q_bytes >= k_bytes) { + n_tok = nt2; + n_qbuf = 2; + } + } + + const uint32_t n_ch = (n_tok * n_head + 31) / 32; + const uint32_t kv_seg = hex_round_up(k->ne[2], 64); + const uint32_t n_rows = n_qsets * kv_seg; + const uint32_t bc_max = n_rows <= 512 ? n_rows : (std::max)(256u, n_rows / 4); + uint32_t kv_block = 64; + while (kv_block < HTP_LID_HMX_KV_BLOCK_MAX && kv_block < bc_max && (kv_block / 32) * n_ch * (n_dot + 1) < 256 && + vtcm_usage(n_tok, kv_block * 2, n_qsets * n_qbuf) <= htp_session_vtcm_size(sess)) { + kv_block *= 2; + } + + const size_t vtcm_size = vtcm_usage(n_tok, kv_block, n_qsets * n_qbuf); + if (vtcm_size <= htp_session_vtcm_size(sess)) { + const uint32_t n_tt = (n_tokens + n_tok - 1) / n_tok; + kparams->kernel_type = HTP_LID_KERNEL_HMX; + kparams->n_threads = n_threads; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = (n_rows + kv_block - 1) / kv_block; + kparams->blocks_per_thread = 0; + kparams->kv_block = kv_block; + kparams->n_qsets = n_qsets; + kparams->n_qbuf = n_qbuf; + kparams->kv_seg = kv_seg; + kparams->n_tt = n_tt; + kparams->vtcm_size = (uint32_t) vtcm_size; + kparams->div_kv_seg = init_fastdiv_values(kv_seg); + kparams->div_n_tt = init_fastdiv_values(n_tt); + kparams->div_n_ch = init_fastdiv_values(n_ch); + return true; + } + } + + const uint32_t n_threads = (std::min)(htp_session_n_threads(sess), n_kv_blocks); + uint32_t n_tok = (std::min)((uint32_t) q->ne[2], (uint32_t) HTP_LID_TOKENS_MAX); + while (n_tok > 1 && htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size) > htp_session_vtcm_size(sess)) { + n_tok /= 2; + } + + const size_t vtcm_size = htp_lid_compute_vtcm_usage(D, n_head, n_tok, n_threads, k_row_size); + if (vtcm_size > htp_session_vtcm_size(sess)) { + return false; + } + + kparams->kernel_type = HTP_LID_KERNEL_HVX; + kparams->n_threads = n_threads; + kparams->n_tokens_tile = n_tok; + kparams->n_kv_blocks = n_kv_blocks; + kparams->blocks_per_thread = (n_kv_blocks + n_threads - 1) / n_threads; + kparams->kv_block = HTP_LID_KV_BLOCK; + kparams->n_qsets = 1; + kparams->n_qbuf = 1; + kparams->kv_seg = 0; + kparams->n_tt = 0; + kparams->vtcm_size = (uint32_t) vtcm_size; + + return true; +} + +static bool htp_kparams_solve_lightning_indexer( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_lid_kernel_params * kparams +) { + if (htp_lightning_indexer_can_collapse(op->src[0], op->src[1], op->src[2], op->src[3], op)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(op->src[0], op->src[1], op->src[2], op->src[3], op, + &q_c, &k_c, &w_c, &m_c, &d_c); + return htp_kparams_solve_lightning_indexer_impl(sess, &q_c, &k_c, &w_c, &m_c, &d_c, kparams); + } + return htp_kparams_solve_lightning_indexer_impl(sess, op->src[0], op->src[1], op->src[2], op->src[3], op, kparams); +} template static inline bool htp_kparams_invoke( const struct ggml_hexagon_session * sess, @@ -4861,6 +5089,28 @@ void htp_opnode_collapse(const struct ggml_hexagon_session * sess, htp_opnode & } return; } + + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + if (node.inputs.size() < 4 || !node.inputs[0] || !node.inputs[1] || !node.inputs[2] || !node.inputs[3] || node.outputs.empty() || !node.outputs[0]) { + return; + } + const struct ggml_tensor * q = node.inputs[0]; + const struct ggml_tensor * k = node.inputs[1]; + const struct ggml_tensor * w = node.inputs[2]; + const struct ggml_tensor * m = node.inputs[3]; + const struct ggml_tensor * dst = node.outputs[0]; + + if (htp_lightning_indexer_can_collapse(q, k, w, m, dst)) { + struct ggml_tensor q_c, k_c, w_c, m_c, d_c; + htp_lightning_indexer_collapse_tensors(q, k, w, m, dst, &q_c, &k_c, &w_c, &m_c, &d_c); + node.inputs[0] = node.add_dummy(q_c); + node.inputs[1] = node.add_dummy(k_c); + node.inputs[2] = node.add_dummy(w_c); + node.inputs[3] = node.add_dummy(m_c); + node.outputs[0] = node.add_dummy(d_c); + } + return; + } } bool htp_kparams_solve( @@ -4901,6 +5151,11 @@ bool htp_kparams_solve( const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; return htp_kparams_solve_dsv4_hc_post_impl(sess, src0, src1, src2, src3, dst, node.kparams()); } + if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const ggml_tensor * src2 = node.inputs.size() > 2 ? node.inputs[2] : nullptr; + const ggml_tensor * src3 = node.inputs.size() > 3 ? node.inputs[3] : nullptr; + return htp_kparams_solve_lightning_indexer_impl(sess, src0, src1, src2, src3, dst, node.kparams()); + } return htp_kparams_solve(sess, node.node, node.kernel_params); } @@ -5005,6 +5260,9 @@ bool htp_kparams_solve( case HTP_OP_ROLL: return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_LIGHTNING_INDEXER: + return htp_kparams_invoke(sess, op, kparams); + case HTP_OP_DIAG: return htp_kparams_invoke(sess, op, kparams); diff --git a/ggml/src/ggml-hexagon/htp-kparams.h b/ggml/src/ggml-hexagon/htp-kparams.h index 70616fbf5ae8..bac3383142c1 100644 --- a/ggml/src/ggml-hexagon/htp-kparams.h +++ b/ggml/src/ggml-hexagon/htp-kparams.h @@ -18,6 +18,7 @@ #include "htp/get-rows-ops.h" #include "htp/glu-ops.h" #include "htp/im2col-ops.h" +#include "htp/lightning-indexer-ops.h" #include "htp/matmul-ops.h" #include "htp/pad-ops.h" #include "htp/pool-ops.h" diff --git a/ggml/src/ggml-hexagon/htp-opnode.cpp b/ggml/src/ggml-hexagon/htp-opnode.cpp index 6aee385badc4..a4efa850310a 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.cpp +++ b/ggml/src/ggml-hexagon/htp-opnode.cpp @@ -11,6 +11,7 @@ #include "htp/gated-delta-net-ops.h" #include "htp/get-rows-ops.h" #include "htp/glu-ops.h" +#include "htp/lightning-indexer-ops.h" #include "htp/matmul-ops.h" #include "htp/pad-ops.h" #include "htp/repeat-ops.h" @@ -197,6 +198,12 @@ static std::string format_kernel_params(const htp_opnode & node) { const char * ktype_str = kparams->kernel_type == HTP_DSV4_HC_POST_KERNEL_COMB ? "comb" : "identity"; snprintf(str, max_size, "%s nth %u vtcm %u", ktype_str, (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_LIGHTNING_INDEXER) { + const auto * kparams = node.kparams(); + snprintf(str, max_size, "%s tok %u bc %u nth %u vtcm %u", + kparams->kernel_type == HTP_LID_KERNEL_HMX ? "hmx" : "hvx", + (unsigned int) kparams->n_tokens_tile, (unsigned int) kparams->kv_block, + (unsigned int) kparams->n_threads, (unsigned int) kparams->vtcm_size); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { const auto * kparams = node.kparams(); diff --git a/ggml/src/ggml-hexagon/htp/CMakeLists.txt b/ggml/src/ggml-hexagon/htp/CMakeLists.txt index 3dcf4445f603..cf7c2f297f80 100644 --- a/ggml/src/ggml-hexagon/htp/CMakeLists.txt +++ b/ggml/src/ggml-hexagon/htp/CMakeLists.txt @@ -51,6 +51,7 @@ add_library(${HTP_LIB} SHARED upscale-ops.c allreduce-ops.c dsv4-hc-ops.c + lightning-indexer-ops.c ) target_compile_definitions(${HTP_LIB} PRIVATE diff --git a/ggml/src/ggml-hexagon/htp/htp-ctx.h b/ggml/src/ggml-hexagon/htp/htp-ctx.h index e6e6254d2cdb..5c35cd988fa0 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ctx.h +++ b/ggml/src/ggml-hexagon/htp/htp-ctx.h @@ -203,5 +203,6 @@ int op_pool_1d(struct htp_ops_context * octx); int op_allreduce(struct htp_ops_context * octx); int op_roll(struct htp_ops_context * octx); int op_upscale(struct htp_ops_context * octx); +int op_lightning_indexer(struct htp_ops_context * octx); #endif /* HTP_CTX_H */ diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index 99557672ee31..96257c241263 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -120,6 +120,7 @@ enum htp_op_code { HTP_OP_DSV4_HC_PRE, HTP_OP_DSV4_HC_POST, HTP_OP_TOP_K_MASK, + HTP_OP_LIGHTNING_INDEXER, HTP_OP_INVALID }; diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c new file mode 100644 index 000000000000..c8a42eb4e372 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.c @@ -0,0 +1,1722 @@ +#pragma clang diagnostic ignored "-Wunused-variable" +#pragma clang diagnostic ignored "-Wunused-function" +#pragma clang diagnostic ignored "-Wunused-but-set-variable" + +#include +#include + +#include "dma-queue.h" +#include "hex-profile.h" +#include "hvx-utils.h" +#include "hvx-copy.h" +#include "htp-vtcm.h" +#include "work-queue.h" + +#define GGML_COMMON_DECL_C +#include "ggml-common.h" +#include "htp-ctx.h" +#include "htp-ops.h" +#include "htp-tensor.h" +#include "hvx-quant.h" +#include "hvx-fa-kernels.h" +#include "hmx-utils.h" +#include "hmx-fa-kernels.h" +#include "hmx-queue.h" + +#include "lightning-indexer-ops.h" + +struct htp_lid_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + + uint32_t n_blocks; + uint32_t blocks_per_thread; + + // current stream and token tile + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + + struct htp_lid_vtcm_layout L; + + uint8_t * spad_q; + uint8_t * spad_w; + uint8_t * spad_qs; + uint8_t * spad_k; + uint8_t * spad_kt; + uint8_t * spad_m; + uint8_t * spad_o; +}; + +static inline void lid_prep_step_info(uint32_t k, uint32_t K, uint32_t t_first, uint32_t i0, uint32_t i1, uint32_t n_head, + uint32_t * out_t, uint32_t * out_ha, uint32_t * out_hb) { + const uint32_t t = t_first + k; + const uint32_t ha = (k == 0) ? (i0 - t * n_head) : 0; + const uint32_t hb = (k == K - 1) ? (i1 - t * n_head) : n_head; + *out_t = t; + *out_ha = ha; + *out_hb = hb; +} + +static inline void lid_prep_push_step(dma_queue * dma_q, const struct htp_lid_context * lctx, + const struct htp_tensor * q, const struct htp_tensor * w, + uint8_t * qs, uint8_t * ws, uint32_t qs_row_size, uint32_t w_row_size, + uint32_t D, uint32_t n_head, uint32_t t, uint32_t ha, uint32_t hb) { + const uint32_t it = lctx->t0 + t; + dma_queue_push(dma_q, dma_make_data(qs, q->data + it * q->nb[2] + lctx->is * q->nb[3] + ha * q->nb[1]), + qs_row_size, q->nb[1], D * sizeof(float), hb - ha); + dma_queue_push(dma_q, dma_make_data(ws, w->data + it * w->nb[1] + lctx->is * w->nb[3]), + w_row_size, w_row_size, n_head * sizeof(float), 1); +} + +// Splat every Q element (f16) and every head weight (f32) into a full vector. +// Work is split over (token, head) pairs. +static void lid_prep_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + + // nth is min(n_threads, n_items): the same chunk as a split over n_threads + const uint32_t n_items = lctx->nt * n_head; + const uint32_t chunk = fastdiv(n_items + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t i0 = ith * chunk; + const uint32_t i1 = MIN(i0 + chunk, n_items); + + if (i0 >= i1) { + return; + } + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t w_row_size = hex_round_up(n_head * sizeof(float), 128); + const size_t qs_size = n_head * L->size_qs_row; + const size_t slot_size = qs_size + w_row_size; + + uint8_t * qs[2]; + uint8_t * ws[2]; + qs[0] = lctx->spad_qs + ith * L->size_qs; + ws[0] = qs[0] + qs_size; + qs[1] = qs[0] + slot_size; + ws[1] = qs[1] + qs_size; + uint8_t * qh = qs[0] + 2 * slot_size; + + const uint32_t t_first = fastdiv(i0, &lctx->kp->div_n_head); + const uint32_t t_last = fastdiv(i1 - 1, &lctx->kp->div_n_head); + const uint32_t n_steps = t_last - t_first + 1; + + // Prologue: prime both slots + uint32_t t, ha, hb; + lid_prep_step_info(0, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[0], ws[0], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + + if (n_steps > 1) { + lid_prep_step_info(1, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[1], ws[1], L->size_qs_row, w_row_size, D, n_head, t, ha, hb); + } + + // Loop: pop(slot) -> compute(slot) -> push(slot, k + 2) + for (uint32_t k = 0; k < n_steps; ++k) { + const uint32_t slot = k & 1; + + dma_queue_pop(dma_q); + dma_queue_pop(dma_q); + + lid_prep_step_info(k, n_steps, t_first, i0, i1, n_head, &t, &ha, &hb); + const uint32_t it = lctx->t0 + t; + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + HVX_Vector * restrict q_sp = (HVX_Vector *) lctx->spad_q + (t * n_head + ha) * D; + for (uint32_t h = ha; h < hb; ++h) { + hvx_copy_f16_f32_aa(qh, qs[slot] + (h - ha) * L->size_qs_row, D); + + const HVX_Vector * restrict qv = (const HVX_Vector *) qh; + for (uint32_t d0 = 0; d0 < D; d0 += VLEN_FP16) { + const HVX_Vector v = qv[d0 / VLEN_FP16]; + const uint32_t nd = MIN(VLEN_FP16, D - d0); + for (uint32_t j = 0; j < nd; ++j) { + *q_sp++ = hvx_vec_repl_f16(Q6_V_vror_VR(v, j * SIZEOF_FP16)); + } + } + } + + const HVX_Vector * restrict wv = (const HVX_Vector *) ws[slot]; + HVX_Vector * restrict w_dst = (HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + for (uint32_t h = ha; h < hb; ++h) { + w_dst[h] = hvx_vec_repl4(Q6_V_vror_VR(wv[h / VLEN_FP32], (h % VLEN_FP32) * SIZEOF_FP32)); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) it); + + if (k + 2 < n_steps) { + uint32_t next_t, next_ha, next_hb; + lid_prep_step_info(k + 2, n_steps, t_first, i0, i1, n_head, &next_t, &next_ha, &next_hb); + lid_prep_push_step(dma_q, lctx, q, w, qs[slot], ws[slot], L->size_qs_row, w_row_size, D, n_head, next_t, next_ha, next_hb); + } + } +} + +#if __HVX_ARCH__ < 79 +#define LID_MUL_F16(a, b) Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(a, b)) +#else +#define LID_MUL_F16(a, b) Q6_Vhf_vmpy_VhfVhf(a, b) +#endif + +// Q8_0 row to f16 in place, 4 blocks (128 values) per step. +// Go backwards: the f16 output of a group is wider than its input and covers the input of the next group. +static inline void lid_dequant_row_q8_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK8_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred q96 = Q6_Q_vsetq_R(96); + + for (int32_t g = (nb - 1) / 4; g >= 0; --g) { + const uint8_t * src = row + g * 4 * sizeof(block_q8_0); + const HVX_Vector v0 = *(const HVX_UVector *) src; + const HVX_Vector v1 = *(const HVX_UVector *) (src + VLEN); + + HVX_Vector qs = Q6_V_vmux_QVV(q96, Q6_V_valign_VVR(v1, v0, 6), Q6_V_valign_VVR(v1, v0, 8)); + qs = Q6_V_vmux_QVV(q64, Q6_V_valign_VVR(v1, v0, 4), qs); + qs = Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(v1, v0, 2), qs); + + const HVX_Vector s01 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v0), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 34))); + const HVX_Vector s23 = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 68)), hvx_vec_repl_f16(Q6_V_valign_VVR(v1, v0, 102))); + + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(qs); + + HVX_Vector * dst = (HVX_Vector *) (row + g * 2 * VLEN); + dst[0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s01)); + if (g * 4 + 2 < nb) { + dst[1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s23)); + } + } +} + +// Q4_0 blocks i, i + 1 (v loaded at block i) to signed quants: block i in bytes 0..31, block i + 1 in bytes 32..63. +// qs[j] holds value j (low nibble) and j + 16 (high nibble). lo16 selects bytes 0..15 and 32..47. +static inline HVX_Vector lid_q4_0_pair_quants(HVX_Vector v, HVX_VectorPred q32, HVX_VectorPred lo16) { + const HVX_Vector qs = Q6_V_vmux_QVV(q32, Q6_V_vror_VR(v, sizeof(ggml_half)), + Q6_V_vror_VR(v, VLEN - 32 + sizeof(block_q4_0) + sizeof(ggml_half))); + const HVX_Vector lo = Q6_Vb_vsub_VbVb(Q6_V_vand_VV(qs, Q6_Vb_vsplat_R(0x0F)), Q6_Vb_vsplat_R(8)); + const HVX_Vector hi = Q6_Vb_vsub_VbVb(Q6_Vub_vlsr_VubR(qs, 4), Q6_Vb_vsplat_R(8)); + return Q6_V_vmux_QVV(lo16, lo, Q6_V_vlalign_VVR(hi, Q6_V_vzero(), 16)); +} + +// Q4_0 row to f16 in place, 2 blocks (64 values) per step. Backwards, as for Q8_0. +static inline void lid_dequant_row_q4_0(uint8_t * restrict row, uint32_t D) { + const int32_t nb = D / QK4_0; + + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred q64 = Q6_Q_vsetq_R(64); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (int32_t p = (nb - 1) / 2; p >= 0; --p) { + const HVX_Vector v = *(const HVX_UVector *) (row + p * 2 * sizeof(block_q4_0)); + const HVX_Vector q = lid_q4_0_pair_quants(v, q32, lo16); + const HVX_Vector d1 = 2 * p + 1 < nb ? hvx_vec_repl_f16(Q6_V_vror_VR(v, sizeof(block_q4_0))) : Q6_V_vzero(); + const HVX_Vector d = Q6_V_vmux_QVV(q64, hvx_vec_repl_f16(v), d1); + + ((HVX_Vector *) row)[p] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(Q6_Wh_vunpack_Vb(q))), d); + } +} + +// Quantized kv block to f16 rows in place. Not inlined: the dequant constants would stay live across the compute loop. +static __attribute__((noinline)) void lid_dequant_block(uint8_t * restrict k_buf, size_t row_stride, uint32_t nkv, uint32_t D, bool q4) { + if (q4) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q4_0(k_buf + r * row_stride, D); + } + } else { + for (uint32_t r = 0; r < nkv; ++r) { + lid_dequant_row_q8_0(k_buf + r * row_stride, D); + } + } +} + +static inline void lid_convert_row_f32_to_f16(uint8_t * row, uint32_t D) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector * src = (const HVX_Vector *) row; + HVX_Vector * dst = (HVX_Vector *) row; + + for (uint32_t d = 0; d < D; d += 64) { + dst[d / 64] = hvx_vec_f32_to_f16(src[d / 32], d + 32 < D ? src[d / 32 + 1] : zero); + } +} + +// Q8_0 gather transpose limits: tmp (two 64x64 scratch tiles) holds D / 2 pairs, gather offsets are 16 bit +static inline bool lid_q8_0_gather_ok(size_t row_size, uint32_t D) { + return D <= 4 * HTP_LID_KV_BLOCK && (HTP_LID_KV_BLOCK - 1) * row_size + (D / QK8_0) * sizeof(block_q8_0) <= 32767; +} + +// Transpose a raw Q8_0 kv block: one halfword gather reads q[d], q[d + 1] of all 64 rows, then split, widen to f16 and scale. +// Lanes of missing rows hold stale data; they are not stored. +static inline void lid_transpose_q8_0_gather(HVX_Vector * restrict kt, + HVX_Vector * restrict tmp, + HVX_Vector * restrict sc, + const uint8_t * restrict k_buf, + size_t row_size, + HVX_Vector offsets, + uint32_t D) { + const uint32_t mu = HTP_LID_KV_BLOCK * row_size; + for (uint32_t b = 0; b < D / QK8_0; ++b) { + Q6_vgather_ARMVh(sc + b, (size_t) (k_buf + b * sizeof(block_q8_0)), mu, offsets); + } + for (uint32_t d = 0; d < D; d += 2) { + const uint32_t off = (d / QK8_0) * sizeof(block_q8_0) + sizeof(ggml_half) + d % QK8_0; + Q6_vgather_ARMVh(tmp + d / 2, (size_t) (k_buf + off), mu, offsets); + } + hvx_gather_sync(sc); + hvx_gather_sync(tmp); + + for (uint32_t d = 0; d < D; d += 2) { + const HVX_Vector s = sc[d / QK8_0]; + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(tmp[d / 2]); + const HVX_VectorPair qd = Q6_W_vdeal_VVR(Q6_V_hi_W(qh), Q6_V_lo_W(qh), -2); + kt[d + 0] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_lo_W(qd)), s)); + kt[d + 1] = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(Q6_V_hi_W(qd)), s)); + } +} + +static inline void lid_shuff_pass(HVX_Vector * restrict dst, const HVX_Vector * restrict src) { + for (uint32_t j = 0; j < 32; ++j) { + HVX_VectorPair p = Q6_W_vshuff_VVR(src[j + 32], src[j], -SIZEOF_FP16); + dst[2 * j + 0] = Q6_V_lo_W(p); + dst[2 * j + 1] = Q6_V_hi_W(p); + } +} + +// 64x64 f16 transpose: each shuffle pass rotates the 12-bit (row, lane) index by one bit, six passes swap row and lane +static inline void lid_transpose_64x64_f16(HVX_Vector * restrict dst, + const uint8_t * restrict src, + size_t src_stride, + uint32_t nkv, + HVX_Vector * restrict t0, + HVX_Vector * restrict t1) { + if (nkv >= HTP_LID_KV_BLOCK) { + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = *(const HVX_Vector *) (src + j * src_stride); + const HVX_Vector r1 = *(const HVX_Vector *) (src + (j + 32) * src_stride); + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } else { + const HVX_Vector zero = Q6_V_vzero(); + for (uint32_t j = 0; j < 32; ++j) { + const HVX_Vector r0 = (j < nkv) ? *(const HVX_Vector *) (src + j * src_stride) : zero; + const HVX_Vector r1 = (j + 32 < nkv) ? *(const HVX_Vector *) (src + (j + 32) * src_stride) : zero; + HVX_VectorPair p = Q6_W_vshuff_VVR(r1, r0, -SIZEOF_FP16); + t0[2 * j + 0] = Q6_V_lo_W(p); + t0[2 * j + 1] = Q6_V_hi_W(p); + } + } + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(t1, t0); + lid_shuff_pass(t0, t1); + lid_shuff_pass(dst, t0); +} + +#if __HVX_ARCH__ < 79 +// qf32 accumulators, converted to sf once per head +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + HVX_VectorPair m = Q6_Wqf32_vmpy_VhfVhf(x, y); + return Q6_W_vcombine_VV(Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_hi_W(acc), Q6_V_hi_W(m)), + Q6_Vqf32_vadd_Vqf32Vqf32(Q6_V_lo_W(acc), Q6_V_lo_W(m))); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return Q6_Vsf_equals_Vqf32(v); +} +#else +static inline HVX_VectorPair lid_mpyacc(HVX_VectorPair acc, HVX_Vector x, HVX_Vector y) { + return Q6_Wsf_vmpyacc_WsfVhfVhf(acc, x, y); +} +static inline HVX_Vector lid_acc_to_sf(HVX_Vector v) { + return v; +} +#endif + +// out += relu(s) * w, for the even and odd kv lanes +static inline HVX_VectorPair lid_head_acc(HVX_VectorPair out, HVX_VectorPair s, HVX_Vector w) { + const HVX_Vector zero = Q6_V_vzero(); + HVX_Vector e = HVX_OP_ADD_F32(Q6_V_lo_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_lo_W(s)), zero), w)); + HVX_Vector o = HVX_OP_ADD_F32(Q6_V_hi_W(out), HVX_OP_MUL_F32(Q6_Vsf_vmax_VsfVsf(lid_acc_to_sf(Q6_V_hi_W(s)), zero), w)); + return Q6_W_vcombine_VV(o, e); +} + +// Store one output row: (even, odd) lanes back to natural order, add the mask +static inline void lid_store_row(float * restrict o, HVX_VectorPair out, const __fp16 * restrict m) { + const HVX_Vector zero = Q6_V_vzero(); + + const HVX_VectorPair out_nat = Q6_W_vshuff_VVR(Q6_V_hi_W(out), Q6_V_lo_W(out), -SIZEOF_FP32); + + HVX_Vector acc0 = Q6_V_lo_W(out_nat); + HVX_Vector acc1 = Q6_V_hi_W(out_nat); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = *(const HVX_Vector *) m; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, Q6_Vh_vsplat_R(0xFC00)); + + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, Q6_Vh_vsplat_R(0x3C00), zero)); + + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + + acc0 = HVX_OP_ADD_F32(acc0, Q6_V_lo_W(m_f32)); + acc1 = HVX_OP_ADD_F32(acc1, Q6_V_hi_W(m_f32)); + acc0 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + acc1 = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + + HVX_Vector * restrict vo = (HVX_Vector *) o; + vo[0] = acc0; + vo[1] = acc1; +} + +// One output row (token) of a kv block: sum_h relu(q_h . k) * w_h + mask +// kt holds the K block transposed (one f16 vector of 64 kv per d), q_t holds the Q splats of the token. +static inline void lid_compute_row(float * restrict o, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt, + const __fp16 * restrict m, + uint32_t D, + uint32_t n_head) { + // f16 x f16 products land in (even kv, odd kv) lane order + HVX_VectorPair out = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair s0 = Q6_W_vzero(); + HVX_VectorPair s1 = Q6_W_vzero(); + HVX_VectorPair s2 = Q6_W_vzero(); + HVX_VectorPair s3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k = kt[d]; + s0 = lid_mpyacc(s0, k, q0[d]); + s1 = lid_mpyacc(s1, k, q1[d]); + s2 = lid_mpyacc(s2, k, q2[d]); + s3 = lid_mpyacc(s3, k, q3[d]); + } + + out = lid_head_acc(out, s0, w_t[h + 0]); + out = lid_head_acc(out, s1, w_t[h + 1]); + out = lid_head_acc(out, s2, w_t[h + 2]); + out = lid_head_acc(out, s3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair s0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + s0 = lid_mpyacc(s0, kt[d], q0[d]); + } + out = lid_head_acc(out, s0, w_t[h]); + } + + lid_store_row(o, out, m); +} + +// Same as lid_compute_row for two kv blocks at once: each Q splat load feeds both blocks +static inline void lid_compute_row2(float * restrict o0, + float * restrict o1, + const HVX_Vector * restrict q_t, + const HVX_Vector * restrict w_t, + const HVX_Vector * restrict kt0, + const HVX_Vector * restrict kt1, + const __fp16 * restrict m0, + const __fp16 * restrict m1, + uint32_t D, + uint32_t n_head) { + HVX_VectorPair out0 = Q6_W_vzero(); + HVX_VectorPair out1 = Q6_W_vzero(); + + uint32_t h = 0; + for (; h + 4 <= n_head; h += 4) { + const HVX_Vector * restrict q0 = q_t + h * D; + const HVX_Vector * restrict q1 = q0 + D; + const HVX_Vector * restrict q2 = q1 + D; + const HVX_Vector * restrict q3 = q2 + D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair a1 = Q6_W_vzero(); + HVX_VectorPair a2 = Q6_W_vzero(); + HVX_VectorPair a3 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + HVX_VectorPair b1 = Q6_W_vzero(); + HVX_VectorPair b2 = Q6_W_vzero(); + HVX_VectorPair b3 = Q6_W_vzero(); + + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector k0 = kt0[d]; + const HVX_Vector k1 = kt1[d]; + const HVX_Vector v0 = q0[d]; + const HVX_Vector v1 = q1[d]; + const HVX_Vector v2 = q2[d]; + const HVX_Vector v3 = q3[d]; + a0 = lid_mpyacc(a0, k0, v0); + b0 = lid_mpyacc(b0, k1, v0); + a1 = lid_mpyacc(a1, k0, v1); + b1 = lid_mpyacc(b1, k1, v1); + a2 = lid_mpyacc(a2, k0, v2); + b2 = lid_mpyacc(b2, k1, v2); + a3 = lid_mpyacc(a3, k0, v3); + b3 = lid_mpyacc(b3, k1, v3); + } + + out0 = lid_head_acc(out0, a0, w_t[h + 0]); + out0 = lid_head_acc(out0, a1, w_t[h + 1]); + out0 = lid_head_acc(out0, a2, w_t[h + 2]); + out0 = lid_head_acc(out0, a3, w_t[h + 3]); + out1 = lid_head_acc(out1, b0, w_t[h + 0]); + out1 = lid_head_acc(out1, b1, w_t[h + 1]); + out1 = lid_head_acc(out1, b2, w_t[h + 2]); + out1 = lid_head_acc(out1, b3, w_t[h + 3]); + } + for (; h < n_head; ++h) { + const HVX_Vector * restrict q0 = q_t + h * D; + + HVX_VectorPair a0 = Q6_W_vzero(); + HVX_VectorPair b0 = Q6_W_vzero(); + for (uint32_t d = 0; d < D; ++d) { + const HVX_Vector v0 = q0[d]; + a0 = lid_mpyacc(a0, kt0[d], v0); + b0 = lid_mpyacc(b0, kt1[d], v0); + } + out0 = lid_head_acc(out0, a0, w_t[h]); + out1 = lid_head_acc(out1, b0, w_t[h]); + } + + lid_store_row(o0, out0, m0); + lid_store_row(o1, out1, m1); +} + +// Each thread owns a range of kv blocks and streams K / mask in, output out (HTP_LID_N_BUF buffers) +static void lid_compute_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_context * lctx = (struct htp_lid_context *) data; + const struct htp_ops_context * octx = lctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_tensor * dst = octx->dst; + + const uint32_t b0 = lctx->blocks_per_thread * ith; + const uint32_t b1 = MIN(b0 + lctx->blocks_per_thread, lctx->n_blocks); + + if (b0 >= b1) { + return; + } + + const struct htp_lid_vtcm_layout * L = &lctx->L; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_kv = k->ne[2]; + const uint32_t nt = lctx->nt; + + const bool is_f32 = (k->type == HTP_TYPE_F32); + const bool is_q8_0 = (k->type == HTP_TYPE_Q8_0); + const bool is_q4_0 = (k->type == HTP_TYPE_Q4_0); + const uint32_t k_row_size = lctx->kp->k_row_size; + + const uint32_t nbk2 = k->nb[2]; + const uint32_t nbm1 = m->nb[1]; + const uint32_t nb1 = dst->nb[1]; + + const dma_addr_t k_base = k->data + lctx->is * k->nb[3]; + const dma_addr_t m_base = m->data + lctx->t0 * nbm1 + lctx->im3 * m->nb[3]; + const dma_addr_t d_base = dst->data + lctx->t0 * nb1 + lctx->is * dst->nb[3]; + + uint8_t * spad_k = lctx->spad_k + ith * HTP_LID_N_BUF * L->size_k_buf; + uint8_t * spad_m = lctx->spad_m + ith * HTP_LID_N_BUF * L->size_m_buf; + uint8_t * spad_o = lctx->spad_o + ith * HTP_LID_N_BUF * L->size_o_buf; + + HVX_Vector * kt[2]; + kt[0] = (HVX_Vector *) (lctx->spad_kt + ith * L->size_kt); + kt[1] = kt[0] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t0 = kt[1] + hex_round_up(D, HTP_LID_KV_BLOCK); + HVX_Vector * kt_t1 = kt_t0 + HTP_LID_KV_BLOCK; + + // Q8_0: transpose by gather from the raw rows (sc: one vector of block scales per 32 d) + const bool q8_g = is_q8_0 && lid_q8_0_gather_ok(L->size_k_raw_row, D); + HVX_Vector * sc = kt_t1 + HTP_LID_KV_BLOCK; + const HVX_Vector g_offs = hvx_vec_gather_offsets_h(L->size_k_raw_row); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; + + for (uint32_t b = b0, i = 0; b < b1 && i < HTP_LID_N_BUF; ++b, ++i) { + const uint32_t kv0 = b * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base, spad_o + i * L->size_o_buf), + nb1, L->size_o_row, nkv * sizeof(float), 0); + dma_queue_push(dma_q, dma_make_data(spad_k + i * L->size_k_buf, k_base + kv0 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv); + dma_queue_push(dma_q, dma_make_data(spad_m + i * L->size_m_buf, m_base + kv0 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv * sizeof(__fp16), nt); + } + + // Two kv blocks per step, the next two are in flight + for (uint32_t b = b0; b < b1; b += 2) { + const uint32_t nb = MIN(2, b1 - b); + + uint8_t * o_buf[2]; + uint8_t * k_buf[2]; + uint8_t * m_buf[2]; + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - (b + j) * HTP_LID_KV_BLOCK); + + o_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; + k_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + m_buf[j] = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; + + if ((is_q8_0 && !q8_g) || is_q4_0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + lid_dequant_block(k_buf[j], L->size_k_raw_row, nkv, D, is_q4_0); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_DEQUANT, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + if (is_f32) { + for (uint32_t r = 0; r < nkv; ++r) { + lid_convert_row_f32_to_f16(k_buf[j] + r * L->size_k_raw_row, D); + } + } + if (q8_g) { + lid_transpose_q8_0_gather(kt[j], kt_t0, sc, k_buf[j], L->size_k_raw_row, g_offs, D); + } else { + for (uint32_t d0 = 0; d0 < D; d0 += HTP_LID_KV_BLOCK) { + lid_transpose_64x64_f16(kt[j] + d0, k_buf[j] + d0 * sizeof(__fp16), L->size_k_raw_row, nkv, kt_t0, kt_t1); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (b + j)); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + for (uint32_t t = 0; t < nt; ++t) { + const HVX_Vector * q_t = (const HVX_Vector *) lctx->spad_q + t * n_head * D; + const HVX_Vector * w_t = (const HVX_Vector *) (lctx->spad_w + t * n_head * VLEN); + if (nb == 2) { + lid_compute_row2((float *) (o_buf[0] + t * L->size_o_row), (float *) (o_buf[1] + t * L->size_o_row), + q_t, w_t, kt[0], kt[1], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), (const __fp16 *) (m_buf[1] + t * L->size_m_row), + D, n_head); + } else { + lid_compute_row((float *) (o_buf[0] + t * L->size_o_row), q_t, w_t, kt[0], + (const __fp16 *) (m_buf[0] + t * L->size_m_row), D, n_head); + } + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) b); + + for (uint32_t j = 0; j < nb; ++j) { + const uint32_t kv0 = (b + j) * HTP_LID_KV_BLOCK; + const uint32_t nkv = MIN(HTP_LID_KV_BLOCK, n_kv - kv0); + + dma_queue_push(dma_q, dma_make_data(d_base + kv0 * sizeof(float), o_buf[j]), + nb1, L->size_o_row, nkv * sizeof(float), nt); + + if (b + j + HTP_LID_N_BUF < b1) { + const uint32_t kv2 = kv0 + HTP_LID_N_BUF * HTP_LID_KV_BLOCK; + const uint32_t nkv2 = MIN(HTP_LID_KV_BLOCK, n_kv - kv2); + + dma_queue_push(dma_q, dma_make_data(k_buf[j], k_base + kv2 * nbk2), + L->size_k_raw_row, nbk2, k_row_size, nkv2); + dma_queue_push(dma_q, dma_make_data(m_buf[j], m_base + kv2 * sizeof(__fp16)), + L->size_m_row, nbm1, nkv2 * sizeof(__fp16), nt); + } + } + } + + for (uint32_t i = 0; i < MIN(b1 - b0, HTP_LID_N_BUF); ++i) { + dma_queue_pop(dma_q); + } + + (void) nth; +} + +// ----- HMX path: K is the activation, Q the weights, head sum by a second HMX pass ----- +// +// Channel c = t * n_head + h of a token tile. Per kv row tile (32 kv) and token tile u (32 tokens): +// mm1: S[kv, c] = relu(k . q_c) * |w_c| (convert scale |w_c|, pos clamp) +// mm2: O[kv, t] = sum_c S[kv, c] * G[c, t], G[c, t] = sign(w_c) for c of token t +// O tiles hold f16 (kv pair, token) words; gather one token column, add the mask, store f32. +// +// A run preps the Q sets of up to n_qsets (stream, token tile) pairs, then pipelines all their kv rows. +// Row x of a run is kv x % kv_seg of set x / kv_seg (kv_seg = n_kv padded to 64), so a block of Bc rows can +// span sets: it is split in segments, one per set. HMX computes block g while HVX preps K of g + 1 and extracts O of g - 1. + +struct lid_hmx_set { + uint32_t is; + uint32_t im3; + uint32_t t0; + uint32_t nt; + uint32_t n_ch; + uint32_t n_u; +}; + +struct htp_lid_hmx_context { + const struct htp_ops_context * octx; + const struct htp_lid_kernel_params * kp; + struct htp_lid_hmx_vtcm_layout L; + + uint32_t Bc; + uint32_t kv_seg; // rows per set: n_kv padded to 64 + uint32_t n_blocks; // per run + uint32_t n_tile; // tokens per token tile + uint32_t n_tt; // token tiles per stream + + uint32_t n_qsets; // Q sets per buffer + uint32_t qbuf; // Q buffer of the run (two buffers: the next run's Q is prepped during this run) + + uint32_t tile0; // first (stream, token tile) of the run + uint32_t n_run; // sets in the run + uint32_t n_qthr; // threads with Q work in the run + uint32_t nt_run; // tokens of each set in the run + uint32_t n_u_run; // token tiles of 32 of each set in the run + int32_t g; // current stage + + struct lid_hmx_set sets[HTP_LID_HMX_MAX_QSETS]; + + // next run, Q prepped in the stages of this one + bool nx_on; + uint32_t nx_n_run; + struct fastdiv_values nx_stg_div; // stages of this run: n_blocks + 1 + struct lid_hmx_set nx_sets[HTP_LID_HMX_MAX_QSETS]; + + __fp16 * qw; + __fp16 * gw; + uint8_t * sc; + uint8_t * sc_id; + __fp16 * s; + float * w; + uint8_t * qs; + uint8_t * k_raw[2]; + __fp16 * k_t[2]; + __fp16 * o[2]; + uint8_t * m[4]; + uint8_t * out[2]; +}; + +#define LID_HMX_MAX_SEGS HTP_LID_HMX_MAX_QSETS + +// Rows of one set in a block: valid rows [x0, x0 + nv) of the block are kv [kv0, kv0 + nv) of the set +struct lid_hmx_seg { + uint32_t set; + uint32_t x0; + uint32_t kv0; + uint32_t nv; +}; + +static inline struct lid_hmx_set lid_hmx_set_init(const struct htp_lid_hmx_context * hctx, uint32_t set) { + const struct htp_ops_context * octx = hctx->octx; + const uint32_t ti = hctx->tile0 + set; + + struct lid_hmx_set S; + S.is = fastdiv(ti, &hctx->kp->div_n_tt); + S.im3 = fastmodulo(S.is, octx->src[3]->ne[3], &hctx->kp->div_m_ne3); + S.t0 = (ti - S.is * hctx->n_tt) * hctx->n_tile; + S.nt = MIN(hctx->n_tile, octx->src[0]->ne[2] - S.t0); + S.n_ch = (S.nt * octx->src[0]->ne[1] + 31) / 32; + S.n_u = (S.nt + 31) / 32; + return S; +} + +// Segments of block g with valid rows. Returns their count, n_rows gets the block rows (multiple of 64). +static inline uint32_t lid_hmx_segs(const struct htp_lid_hmx_context * hctx, uint32_t g, struct lid_hmx_seg * seg, uint32_t * n_rows) { + const uint32_t n_kv = hctx->octx->src[1]->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t xb = MIN(xa + hctx->Bc, hctx->n_run * ks); + + uint32_t n = 0; + for (uint32_t x = xa; x < xb;) { + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(xb, (set + 1) * ks); + if (kv < n_kv) { + seg[n].set = set; + seg[n].x0 = x - xa; + seg[n].kv0 = kv; + seg[n].nv = MIN(n_kv, kv + (end - x)) - kv; + n++; + } + x = end; + } + *n_rows = xb - xa; + return n; +} + +typedef struct { + const __fp16 * qw; + const __fp16 * gw; + const uint8_t * sc; + uint32_t rt0; + uint32_t n_rt; +} lid_hmx_job_seg_t; + +typedef struct { + lid_hmx_job_seg_t seg[LID_HMX_MAX_SEGS]; + uint32_t n_seg; + + const __fp16 * k_t; + const uint8_t * sc_id; + __fp16 * s; + __fp16 * o; + uint32_t n_ch; + uint32_t n_u; + uint32_t n_dot; + uint32_t n_head; + uint32_t n_sg; +} lid_hmx_job_t; + +// acc += sum of n tile products, a and b advance one tile per step +static inline void lid_hmx_dot(const __fp16 * a, const __fp16 * b, uint32_t n) { + for (; n > 32; n -= 32) { + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(65535), "r"(a), "r"(b)); + a += 32 * HMX_FP16_TILE_N_ELMS; + b += 32 * HMX_FP16_TILE_N_ELMS; + } + asm volatile(HMX_LOAD_MPY_DEEP_F16("%1", "%2", "%0") :: "r"(n * HMX_FP16_TILE_SIZE - 1), "r"(a), "r"(b)); +} + +// Per segment: channel tile outer, row tile inner, one bias load per channel tile. S holds [n_rt][n_sg] tiles. +static void lid_hmx_worker(void * data) { + const lid_hmx_job_t * job = (const lid_hmx_job_t *) data; + + const size_t dot_stride = (size_t) job->n_dot * HMX_FP16_TILE_N_ELMS; + const size_t s_stride = (size_t) job->n_sg * HMX_FP16_TILE_N_ELMS; + const uint32_t nh = job->n_head; + + for (uint32_t i = 0; i < job->n_seg; ++i) { + const lid_hmx_job_seg_t * sg = &job->seg[i]; + const uint32_t n_rt = sg->n_rt; + const __fp16 * kt = job->k_t + sg->rt0 * dot_stride; + __fp16 * ot = job->o + sg->rt0 * job->n_u * HMX_FP16_TILE_N_ELMS; + + for (uint32_t u = 0; u < job->n_u; ++u) { + const uint32_t j0 = u * nh; + const uint32_t j1 = MIN(j0 + nh, job->n_ch); + + for (uint32_t c = j0; c < j1; ++c) { + const __fp16 * qw = sg->qw + c * dot_stride; + __fp16 * st = job->s + (c - j0) * HMX_FP16_TILE_N_ELMS; + + asm volatile(HMX_SET_BIAS("%0") :: "r"(sg->sc + c * 256)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(kt + rt * dot_stride, qw, job->n_dot); + asm volatile("mxmem(%0, %1):after:pos.hf = acc\n" :: "r"(st + rt * s_stride), "r"(0) : "memory"); + } + } + + asm volatile(HMX_SET_BIAS("%0") :: "r"(job->sc_id)); + for (uint32_t rt = 0; rt < n_rt; ++rt) { + asm volatile(HMX_CLRACC_F16()); + lid_hmx_dot(job->s + rt * s_stride, sg->gw + j0 * HMX_FP16_TILE_N_ELMS, j1 - j0); + asm volatile(HMX_STORE_AFTER_F16("%0", "%1") :: "r"(ot + (rt * job->n_u + u) * HMX_FP16_TILE_N_ELMS), "r"(0) : "memory"); + } + } + } +} + +// Thread slices: K rows by row pairs of the padded block, mask / output rows by token. +// Block 0 is prepped together with Q: its K rows go to the threads without Q work. +// K rows are split only in stages on all n_threads threads. +static inline void lid_hmx_rows(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, uint32_t n_rows, + uint32_t * r0, uint32_t * r1) { + uint32_t per; + if (g == 0 && hctx->n_qthr < nth) { + if (ith < hctx->n_qthr) { + *r0 = *r1 = 0; + return; + } + ith -= hctx->n_qthr; + per = (n_rows + nth - hctx->n_qthr - 1) / (nth - hctx->n_qthr); + } else { + per = fastdiv(n_rows + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div); + } + per = hex_round_up(per, 2); + *r0 = MIN(ith * per, n_rows); + *r1 = MIN(*r0 + per, n_rows); +} + +// Split over n_threads; a stage on one thread has one token, which this split also gives to thread 0 +static inline void lid_hmx_toks(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nt, uint32_t * t0, uint32_t * t1) { + const uint32_t per = fastdiv(nt + hctx->octx->n_threads - 1, &hctx->octx->n_threads_div); + *t0 = MIN(ith * per, nt); + *t1 = MIN((ith + 1) * per, nt); +} + +// Push (or only count, push == false) the K row and mask DMAs of block g for this thread, one per segment +static uint32_t lid_hmx_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g, bool push) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * k = octx->src[1]; + const struct htp_tensor * m = octx->src[3]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + uint32_t r0, r1, ta, tb; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + + dma_queue * dma_q = octx->ctx->dma[ith]; + const uint32_t k_row_size = hctx->kp->k_row_size; + + uint32_t n = 0; + for (uint32_t i = 0; i < n_seg; ++i) { + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + + const uint32_t ra = MAX(r0, seg[i].x0); + const uint32_t rb = MIN(r1, seg[i].x0 + seg[i].nv); + if (ra < rb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->k_raw[g % 2] + ra * L->size_k_raw_row, + k->data + S.is * k->nb[3] + (seg[i].kv0 + ra - seg[i].x0) * k->nb[2]), + L->size_k_raw_row, k->nb[2], k_row_size, rb - ra); + } + n++; + } + if (ta < tb) { + if (push) { + dma_queue_push(dma_q, dma_make_data(hctx->m[g % 4] + ta * L->size_m_row + seg[i].x0 * sizeof(__fp16), + m->data + S.im3 * m->nb[3] + (S.t0 + ta) * m->nb[1] + seg[i].kv0 * sizeof(__fp16)), + L->size_m_row, m->nb[1], seg[i].nv * sizeof(__fp16), tb - ta); + } + n++; + } + } + return n; +} + +static inline uint32_t lid_hmx_push_in(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + return lid_hmx_in(hctx, ith, nth, g, true); +} + +// Row pair (a, b) quants of two blocks (64 values each, bytes 0..63) to the two tile vectors, scaled by s0, s1 +static inline void lid_q8_0_pair_store(HVX_Vector * restrict dst, HVX_Vector a, HVX_Vector b, HVX_Vector s0, HVX_Vector s1, bool two) { + const HVX_VectorPair qh = Q6_Wh_vunpack_Vb(Q6_V_lo_W(Q6_W_vshuff_VVR(b, a, -1))); + dst[0] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_lo_W(qh)), s0); + if (two) { + dst[16] = LID_MUL_F16(Q6_Vhf_equals_Vh(Q6_V_hi_W(qh)), s1); + } +} + +// Raw Q8_0 rows [r0, r1) to activation tiles, dequant fused with the row pair interleave. Rows >= nv are zero. +// Per row pair and two blocks: interleave the quants of both rows as bytes, widen, then scale by the (d_r, d_r+1) word. +static inline void lid_hmx_prep_k_q8_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + if (n_dot == 4) { + // D = 128: the row is 136 bytes, two aligned loads; scales at halfwords 0, 17, 34, 51 + const HVX_Vector a0 = v0 ? ((const HVX_Vector *) p0)[0] : zero; + const HVX_Vector a1 = v0 ? ((const HVX_Vector *) p0)[1] : zero; + const HVX_Vector b0 = v1 ? ((const HVX_Vector *) p1)[0] : zero; + const HVX_Vector b1 = v1 ? ((const HVX_Vector *) p1)[1] : zero; + + const HVX_VectorPair ds = Q6_W_vshuff_VVR(b0, a0, -SIZEOF_FP16); + + lid_q8_0_pair_store(dst, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a0, a0, 2), Q6_V_valign_VVR(a0, a0, 4)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b0, b0, 2), Q6_V_valign_VVR(b0, b0, 4)), + hvx_vec_repl4(Q6_V_lo_W(ds)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_lo_W(ds), 17 * 4)), true); + lid_q8_0_pair_store(dst + 32, + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(a1, a0, 70), Q6_V_valign_VVR(a1, a0, 72)), + Q6_V_vmux_QVV(q32, Q6_V_valign_VVR(b1, b0, 70), Q6_V_valign_VVR(b1, b0, 72)), + hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 2 * 4)), hvx_vec_repl4(Q6_V_vror_VR(Q6_V_hi_W(ds), 19 * 4)), true); + continue; + } + + for (uint32_t i = 0; i < n_dot; i += 2) { + const uint8_t * pa = p0 + i * sizeof(block_q8_0); + const uint8_t * pb = p1 + i * sizeof(block_q8_0); + + // quants of blocks i, i + 1 in bytes 0..63, scales d_i at byte 0 and d_i+1 at byte 34 + const HVX_Vector a = v0 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pa + 2), *(const HVX_UVector *) (pa + 4)) : zero; + const HVX_Vector b = v1 ? Q6_V_vmux_QVV(q32, *(const HVX_UVector *) (pb + 2), *(const HVX_UVector *) (pb + 4)) : zero; + const HVX_Vector da = v0 ? *(const HVX_UVector *) pa : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) pb : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, a, b, hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q8_0))), i + 1 < n_dot); + } + } +} + +// Raw Q4_0 rows [r0, r1) to activation tiles, same as Q8_0: per row pair and two blocks, one load per row. +// Rows >= nv are zero. +static inline void lid_hmx_prep_k_q4_0(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_VectorPred q32 = Q6_Q_vsetq_R(32); + const HVX_VectorPred lo16 = Q6_Q_or_QQ(Q6_Q_vsetq_R(16), Q6_Q_and_QQn(Q6_Q_vsetq_R(48), q32)); + + for (uint32_t r = r0; r < r1; r += 2) { + const uint8_t * p0 = k_raw + r * row_stride; + const uint8_t * p1 = p0 + row_stride; + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + // blocks i, i + 1: scales d_i at byte 0 and d_i+1 at byte 18 + const HVX_Vector da = v0 ? *(const HVX_UVector *) (p0 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector db = v1 ? *(const HVX_UVector *) (p1 + i * sizeof(block_q4_0)) : zero; + const HVX_Vector ds = Q6_V_lo_W(Q6_W_vshuff_VVR(db, da, -SIZEOF_FP16)); + + lid_q8_0_pair_store(dst + i * 16, lid_q4_0_pair_quants(da, q32, lo16), lid_q4_0_pair_quants(db, q32, lo16), + hvx_vec_repl4(ds), hvx_vec_repl4(Q6_V_vror_VR(ds, 2 * sizeof(block_q4_0))), i + 1 < n_dot); + } + } +} + +// F16 rows [r0, r1) to activation tiles: row pairs interleaved. Rows >= nv are zero. +static inline void lid_hmx_prep_k_f16(__fp16 * restrict k_t, const uint8_t * restrict k_raw, size_t row_stride, + uint32_t r0, uint32_t r1, uint32_t nv, uint32_t n_dot) { + const HVX_Vector zero = Q6_V_vzero(); + + for (uint32_t r = r0; r < r1; r += 2) { + const HVX_Vector * p0 = (const HVX_Vector *) (k_raw + r * row_stride); + const HVX_Vector * p1 = (const HVX_Vector *) (k_raw + (r + 1) * row_stride); + const bool v0 = r < nv; + const bool v1 = r + 1 < nv; + + HVX_Vector * dst = (HVX_Vector *) (k_t + (r / 32) * n_dot * HMX_FP16_TILE_N_ELMS) + (r % 32) / 2; + + for (uint32_t i = 0; i < n_dot; i += 2) { + const HVX_VectorPair p = Q6_W_vshuff_VVR(v1 ? p1[i / 2] : zero, v0 ? p0[i / 2] : zero, -SIZEOF_FP16); + dst[i * 16] = Q6_V_lo_W(p); + if (i + 1 < n_dot) { + dst[(i + 1) * 16] = Q6_V_hi_W(p); + } + } + } +} + +// K rows of block g (this thread's rows) to activation tiles, one piece per set; set padding rows are zero. +// Not inlined: keeps the stage's registers out of the prep loops. +static __attribute__((noinline)) void lid_hmx_prep_k(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const struct htp_tensor * k = hctx->octx->src[1]; + + const uint32_t n_kv = k->ne[2]; + const uint32_t ks = hctx->kv_seg; + const uint32_t xa = g * hctx->Bc; + const uint32_t n_rows = MIN(hctx->Bc, hctx->n_run * ks - xa); + + uint32_t r0, r1; + lid_hmx_rows(hctx, ith, nth, g, n_rows, &r0, &r1); + + uint8_t * k_raw = hctx->k_raw[g % 2]; + __fp16 * k_t = hctx->k_t[g % 2]; + const bool f32 = k->type == HTP_TYPE_F32; + const bool q8 = k->type == HTP_TYPE_Q8_0; + const bool q4 = k->type == HTP_TYPE_Q4_0; + + for (uint32_t r = r0; r < r1;) { + const uint32_t x = xa + r; + const uint32_t set = fastdiv(x, &hctx->kp->div_kv_seg); + const uint32_t kv = x - set * ks; + const uint32_t end = MIN(r1, (set + 1) * ks - xa); + const uint32_t nv = r + (kv < n_kv ? n_kv - kv : 0); + + if (q8) { + lid_hmx_prep_k_q8_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else if (q4) { + lid_hmx_prep_k_q4_0(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } else { + if (f32) { + for (uint32_t i = r; i < end; ++i) { + lid_convert_row_f32_to_f16(k_raw + i * L->size_k_raw_row, k->ne[0]); + } + } + lid_hmx_prep_k_f16(k_t, k_raw, L->size_k_raw_row, r, end, nv, L->n_dot); + } + r = end; + } +} + +// O column of each token (64 kv per gather) + mask -> f32 rows, then one DMA out per segment +static __attribute__((noinline)) void lid_hmx_extract(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * dst = octx->dst; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return; + } + + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(hctx, g, seg, &n_rows); + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); + + const uint32_t n_u = hctx->n_u_run; + const uint32_t mu = (n_u + 1) * HMX_FP16_TILE_SIZE - 1; + + int32_t offs[32] __attribute__((aligned(128))); + for (int l = 0; l < 32; l++) { + offs[l] = (l / 16) * n_u * HMX_FP16_TILE_SIZE + (l % 16) * 128; + } + const HVX_Vector v_offs = *(const HVX_Vector *) offs; + + HVX_Vector * gbuf = (HVX_Vector *) (hctx->qs + ith * L->size_qs + 2 * 32 * L->size_qs_row + 32 * L->size_qh_row); + + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector ninf = Q6_V_vsplat_R(0xFF800000); + const HVX_Vector one = Q6_Vh_vsplat_R(0x3C00); + const HVX_Vector hinf = Q6_Vh_vsplat_R(0xFC00); + + const __fp16 * o_base = hctx->o[g % 2]; + const uint8_t * m_buf = hctx->m[g % 4]; + uint8_t * o_buf = hctx->out[g % 2]; + + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t q0 = seg[i].x0 / 64; + const uint32_t q1 = (seg[i].x0 + seg[i].nv + 63) / 64; + + for (uint32_t t = ta; t < tb; ++t) { + const uint32_t u = t / 32; + const HVX_Vector off = Q6_Vw_vadd_VwVw(v_offs, Q6_V_vsplat_R((t % 32) * 4)); + + for (uint32_t q = q0; q < q1; ++q) { + const __fp16 * base = o_base + (2 * q * n_u + u) * HMX_FP16_TILE_N_ELMS; + Q6_vgather_ARMVw(gbuf + q - q0, (size_t) base, mu, off); + } + hvx_gather_sync(gbuf + q1 - q0 - 1); + + const HVX_Vector * mv = (const HVX_Vector *) (m_buf + t * L->size_m_row); + HVX_Vector * ov = (HVX_Vector *) (o_buf + t * L->size_o_row); + + for (uint32_t q = q0; q < q1; ++q) { + const HVX_VectorPair s = hvx_vec_f16_to_f32(gbuf[q - q0]); + + // qf32 math does not keep inf: add a finite mask, then put -inf back + const HVX_Vector m_f16 = mv[q]; + const HVX_VectorPred m_inf = Q6_Q_vcmp_eq_VhVh(m_f16, hinf); + const HVX_VectorPair m_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, zero, m_f16)); + const HVX_VectorPair f_f32 = hvx_vec_f16_to_f32(Q6_V_vmux_QVV(m_inf, one, zero)); + + const HVX_Vector acc0 = HVX_OP_ADD_F32(Q6_V_lo_W(s), Q6_V_lo_W(m_f32)); + const HVX_Vector acc1 = HVX_OP_ADD_F32(Q6_V_hi_W(s), Q6_V_hi_W(m_f32)); + ov[2 * q + 0] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_lo_W(f_f32), zero), ninf, acc0); + ov[2 * q + 1] = Q6_V_vmux_QVV(Q6_Q_vcmp_gt_VwVw(Q6_V_hi_W(f_f32), zero), ninf, acc1); + } + } + + const struct lid_hmx_set S = hctx->sets[seg[i].set]; + dma_queue_push(octx->ctx->dma[ith], + dma_make_data(dst->data + S.is * dst->nb[3] + (S.t0 + ta) * dst->nb[1] + seg[i].kv0 * sizeof(float), + o_buf + ta * L->size_o_row + seg[i].x0 * sizeof(float)), + dst->nb[1], L->size_o_row, seg[i].nv * sizeof(float), tb - ta); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_O_PROC, (uint16_t) g); +} + +// Number of output DMAs of block g for this thread +static inline uint32_t lid_hmx_n_out(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth, uint32_t g) { + uint32_t ta, tb; + lid_hmx_toks(hctx, ith, hctx->nt_run, &ta, &tb); + if (ta >= tb) { + return 0; + } + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + return lid_hmx_segs(hctx, g, seg, &n_rows); +} + +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth); + +// Pipeline stage g (HMX runs block g meanwhile): prep block g + 1, extract block g - 1 +static void lid_hmx_stage(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) { + const int32_t g = hctx->g; + const int32_t n = (int32_t) hctx->n_blocks; + + struct htp_thread_trace * tr = &hctx->octx->ctx->trace[ith]; + dma_queue * dma_q = hctx->octx->ctx->dma[ith]; + + // FIFO order per thread: OUT(g - 3), IN(g + 1), OUT(g - 2). When this stage preps next run Q, also pop + // OUT(g - 2) so the queue is empty for the Q DMAs; else leave it in flight for one more stage. + const int32_t lag = hctx->nx_on ? 2 : 3; + + uint32_t n_pop = 0; + if (lag == 3 && g - 3 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 3); + } + if (g >= 0 && g + 1 < n) { + n_pop += lid_hmx_in(hctx, ith, nth, g + 1, false); + } + if (lag == 2 && g - 2 >= 0) { + n_pop += lid_hmx_n_out(hctx, ith, nth, g - 2); + } + for (uint32_t i = 0; i < n_pop; ++i) { + dma_queue_pop(dma_q); + } + + if (hctx->nx_on && g >= 0) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + lid_hmx_q_chunk(hctx, ith, nth); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) g); + } + if (g + 2 < n) { + lid_hmx_push_in(hctx, ith, nth, g + 2); + } + if (g + 1 < n) { + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + lid_hmx_prep_k(hctx, ith, nth, g + 1); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_W_PREP, (uint16_t) (g + 1)); + } + if (g - 1 >= 0) { + lid_hmx_extract(hctx, ith, nth, g - 1); + } + if (g == n) { + uint32_t n_tail = lid_hmx_n_out(hctx, ith, nth, n - 1); + if (lag == 3 && n >= 2) { + n_tail += lid_hmx_n_out(hctx, ith, nth, n - 2); + } + for (uint32_t i = 0; i < n_tail; ++i) { + dma_queue_pop(dma_q); + } + } +} + +static void lid_hmx_stage_thread(unsigned int nth, unsigned int ith, void * data) { + lid_hmx_stage((const struct htp_lid_hmx_context *) data, ith, nth); +} + +// Push the Q rows of channel tile c of set S, one DMA per token. Returns the number of DMA descriptors. +static uint32_t lid_hmx_push_q(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, uint8_t * qs, + const struct lid_hmx_set * S, uint32_t c) { + const struct htp_tensor * q = hctx->octx->src[0]; + const uint32_t nh = q->ne[1]; + const uint32_t ch = c * 32; + const uint32_t nr = MIN(32, S->nt * nh - ch); + + // channel ch + r is (token t, head h); a DMA ends at the token end, so the next one starts at h 0 + uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + uint32_t h = ch - t * nh; + uint32_t n = 0; + for (uint32_t r = 0; r < nr; ++t, h = 0) { + const uint32_t cnt = MIN(nh - h, nr - r); + dma_queue_push(dma_q, dma_make_data(qs + r * hctx->L.size_qs_row, + q->data + S->is * q->nb[3] + (S->t0 + t) * q->nb[2] + h * q->nb[1]), + hctx->L.size_qs_row, q->nb[1], q->ne[0] * sizeof(float), cnt); + r += cnt; + n++; + } + return n; +} + +// Scales (|w| per channel, f16 in the low half of each word) and G tile (sign(w) at row c, column token(c) % 32) +// of one channel tile. w points at the tile's 32 channels, the first one is (token t, head h). +static inline void lid_hmx_build_sc_g(uint8_t * restrict sc, __fp16 * restrict gt, const float * restrict w, + uint32_t t, uint32_t h, uint32_t nr, uint32_t nh) { + const HVX_Vector zero = Q6_V_vzero(); + const HVX_Vector wv = Q6_V_vmux_QVV(Q6_Q_vsetq2_R(nr * sizeof(float)), *(const HVX_Vector *) w, zero); + + HVX_Vector * scv = (HVX_Vector *) sc; + scv[0] = hvx_vec_f32_to_f16_shuff(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + scv[1] = zero; + + // sign(w) as f16 bits in the low half of each word, then packed: row r in halfword r + const HVX_VectorPred nz = Q6_Q_vcmp_gt_VuwVuw(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x7FFFFFFF)), zero); + const HVX_Vector sgn = Q6_V_vor_VV(Q6_V_vsplat_R(0x3C00), Q6_Vw_vasr_VwR(Q6_V_vand_VV(wv, Q6_V_vsplat_R(0x80000000)), 16)); + const HVX_Vector sv = Q6_Vh_vpacke_VwVw(zero, Q6_V_vmux_QVV(nz, sgn, zero)); + + // token column of each row; -1 for missing rows (only rows 0..31 are used) + int16_t tcol[64] __attribute__((aligned(128))) = { 0 }; + for (uint32_t r = 0; r < 32; ++r) { + tcol[r] = r < nr ? (int16_t) (t % 32) : -1; + if (++h == nh) { + h = 0; + t++; + } + } + const HVX_Vector tv = *(const HVX_Vector *) tcol; + + // halfword 2n and 2n + 1 of a G vector are column n + static const int16_t col[64] __attribute__((aligned(128))) = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, + 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, + 22, 22, 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, 28, 28, 29, 29, 30, 30, 31, 31, + }; + const HVX_Vector cv = *(const HVX_Vector *) col; + + // vector p holds rows 2p, 2p + 1 + HVX_Vector * gv = (HVX_Vector *) gt; + for (uint32_t p = 0; p < 16; ++p) { + const HVX_Vector tp = hvx_vec_repl4(Q6_V_vror_VR(tv, p * 4)); + const HVX_Vector sp = hvx_vec_repl4(Q6_V_vror_VR(sv, p * 4)); + gv[p] = Q6_V_vmux_QVV(Q6_Q_vcmp_eq_VhVh(cv, tp), sp, zero); + } +} + +// Q buffer slot of set +static inline uint32_t lid_hmx_slot(const struct htp_lid_hmx_context * hctx, uint32_t qbuf, uint32_t set) { + return qbuf * hctx->n_qsets + set; +} + +// Scales and G of channel tile c of a Q slot (needs the slot's w) +static __attribute__((noinline)) void lid_hmx_q_scg(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t ch = c * 32; + const uint32_t t = fastdiv(ch, &hctx->kp->div_n_head); + + lid_hmx_build_sc_g(hctx->sc + (slot * L->n_ch + c) * 256, hctx->gw + (slot * L->n_ch + c) * HMX_FP16_TILE_N_ELMS, + hctx->w + slot * (L->size_w / sizeof(float)) + ch, t, ch - t * nh, MIN(32, S->nt * nh - ch), nh); +} + +// Staged f32 Q rows of channel tile c to f16, then to the weight tiles of a Q slot +static __attribute__((noinline)) void lid_hmx_q_tiles(const struct htp_lid_hmx_context * hctx, uint32_t slot, const struct lid_hmx_set * S, uint32_t c, + const uint8_t * qs, uint8_t * qh) { + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t D = hctx->octx->src[0]->ne[0]; + const uint32_t nh = hctx->octx->src[0]->ne[1]; + const uint32_t nr = MIN(32, S->nt * nh - c * 32); + + for (uint32_t r = 0; r < nr; ++r) { + hvx_copy_f16_f32_aa(qh + r * L->size_qh_row, qs + r * L->size_qs_row, D); + } + + // missing channels must be zero (finite): zero the tiles, then scatter only the valid rows + __fp16 * qw = hctx->qw + (slot * L->n_ch + c) * L->n_dot * HMX_FP16_TILE_N_ELMS; + if (nr < 32) { + for (uint32_t i = 0; i < L->n_dot * 16; ++i) { + ((HVX_Vector *) qw)[i] = Q6_V_vzero(); + } + } + hmx_interleave_rows_to_tiles(qw, (const __fp16 *) qh, nr, D, L->size_qh_row / sizeof(__fp16), 0, hex_round_up(nr, 2)); + hvx_gather_sync(qw); +} + +static inline uint32_t lid_hmx_find_valid_gc(const struct htp_lid_hmx_context * hctx, uint32_t gc, uint32_t gc1) { + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + const uint32_t n_ch = hctx->L.n_ch; + while (gc < gc1) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * n_ch; + if (c < hctx->nx_sets[set].n_ch) { + return gc; + } + gc++; + } + return gc1; +} + +static inline uint32_t lid_hmx_q_chunk_push(const struct htp_lid_hmx_context * hctx, dma_queue * dma_q, + uint8_t * qs, uint32_t qb, uint32_t gc) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + const uint32_t nh = octx->src[0]->ne[1]; + + const uint32_t set = fastdiv(gc, &hctx->kp->div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + + const uint32_t slot = lid_hmx_slot(hctx, qb, set); + const uint32_t ch = c * 32; + const uint32_t ta = fastdiv(ch, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(ch + 32, S->nt * nh) - 1, &hctx->kp->div_n_head) + 1; + + float * wd = hctx->w + slot * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S->is * w->nb[3] + (S->t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + return 1 + lid_hmx_push_q(hctx, dma_q, qs, S, c); +} + +// Stage g share of the next run's Q prep (the DMA queue of the thread is empty here) +static __attribute__((noinline)) void lid_hmx_q_chunk(const struct htp_lid_hmx_context * hctx, uint32_t ith, uint32_t nth) { + const struct htp_ops_context * octx = hctx->octx; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t n_tot = hctx->nx_n_run * L->n_ch; + const uint32_t a = fastdiv(hctx->g * n_tot, &hctx->nx_stg_div); + const uint32_t b = fastdiv((hctx->g + 1) * n_tot, &hctx->nx_stg_div); + const uint32_t per = fastdiv(b - a + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t gc0 = MIN(a + ith * per, b); + const uint32_t gc1 = MIN(gc0 + per, b); + const uint32_t qb = hctx->qbuf ^ 1; + + dma_queue * dma_q = octx->ctx->dma[ith]; + uint8_t * qs[2]; + qs[0] = hctx->qs + ith * L->size_qs; + qs[1] = qs[0] + 32 * L->size_qs_row; + uint8_t * qh = qs[0] + 2 * 32 * L->size_qs_row; + + uint32_t gc_cur = lid_hmx_find_valid_gc(hctx, gc0, gc1); + if (gc_cur >= gc1) { + return; + } + + const uint32_t gc_next = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + uint32_t gc_prefetch = gc1; + + uint32_t n_dma[2] = {0, 0}; + n_dma[0] = lid_hmx_q_chunk_push(hctx, dma_q, qs[0], qb, gc_cur); + if (gc_next < gc1) { + n_dma[1] = lid_hmx_q_chunk_push(hctx, dma_q, qs[1], qb, gc_next); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_next + 1, gc1); + } + + uint32_t slot = 0; + while (gc_cur < gc1) { + for (uint32_t i = 0; i < n_dma[slot]; ++i) { + dma_queue_pop(dma_q); + } + + const uint32_t set = fastdiv(gc_cur, &hctx->kp->div_n_ch); + const uint32_t c = gc_cur - set * L->n_ch; + const struct lid_hmx_set * S = &hctx->nx_sets[set]; + const uint32_t qslot = lid_hmx_slot(hctx, qb, set); + + lid_hmx_q_scg(hctx, qslot, S, c); + lid_hmx_q_tiles(hctx, qslot, S, c, qs[slot], qh); + + if (gc_prefetch < gc1) { + n_dma[slot] = lid_hmx_q_chunk_push(hctx, dma_q, qs[slot], qb, gc_prefetch); + gc_prefetch = lid_hmx_find_valid_gc(hctx, gc_prefetch + 1, gc1); + } else { + n_dma[slot] = 0; + } + + gc_cur = lid_hmx_find_valid_gc(hctx, gc_cur + 1, gc1); + slot ^= 1; + } +} + +// Q rows to weight tiles, per channel scales |w| and G tiles for all sets of the run, then stage -1 (prep block 0). +// Q work goes to the first threads (thread 0 starts first), the K rows of block 0 to the others. +static void lid_hmx_qprep_thread(unsigned int nth, unsigned int ith, void * data) { + struct htp_lid_hmx_context * hctx = (struct htp_lid_hmx_context *) data; + const struct htp_ops_context * octx = hctx->octx; + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * w = octx->src[2]; + const struct htp_lid_hmx_vtcm_layout * L = &hctx->L; + + const uint32_t nh = q->ne[1]; + + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + dma_queue * dma_q = octx->ctx->dma[ith]; + + const uint32_t n_tot = hctx->n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + octx->n_threads - 1, &octx->n_threads_div); + const uint32_t gc0 = MIN(ith * per, n_tot); + const uint32_t gc1 = MIN(gc0 + per, n_tot); + + uint8_t * qs = hctx->qs + ith * L->size_qs; + uint8_t * qh = qs + 2 * 32 * L->size_qs_row; + + // all input DMAs in flight at once: w of the sets, Q of the first channel tile, then K and mask of block 0 + uint32_t n_dma = 0; + uint32_t first = gc1; + const struct fastdiv_values * div_n_ch = &hctx->kp->div_n_ch; + for (uint32_t set = fastdiv(gc0, div_n_ch); gc0 < gc1 && set <= fastdiv(gc1 - 1, div_n_ch); ++set) { + const struct lid_hmx_set S = hctx->sets[set]; + + const uint32_t c0 = MAX(gc0, set * L->n_ch) - set * L->n_ch; + const uint32_t c1 = MIN(MIN(gc1, (set + 1) * L->n_ch) - set * L->n_ch, S.n_ch); + if (c0 >= c1) { + continue; + } + first = MIN(first, set * L->n_ch + c0); + + const uint32_t ta = fastdiv(c0 * 32, &hctx->kp->div_n_head); + const uint32_t tb = fastdiv(MIN(c1 * 32, S.nt * nh) - 1, &hctx->kp->div_n_head) + 1; + float * wd = hctx->w + lid_hmx_slot(hctx, hctx->qbuf, set) * (L->size_w / sizeof(float)); + dma_queue_push(dma_q, dma_make_data(wd + ta * nh, w->data + S.is * w->nb[3] + (S.t0 + ta) * w->nb[1]), + nh * sizeof(float), w->nb[1], nh * sizeof(float), tb - ta); + n_dma++; + } + + if (first < gc1) { + const uint32_t set = fastdiv(first, div_n_ch); + const struct lid_hmx_set S = hctx->sets[set]; + n_dma += lid_hmx_push_q(hctx, dma_q, qs, &S, first - set * L->n_ch); + } + + uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0); + + for (uint32_t i = 0; i < n_dma; ++i) { + dma_queue_pop(dma_q); + } + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t gc = first; gc < gc1; ++gc) { + const uint32_t set = fastdiv(gc, div_n_ch); + const uint32_t c = gc - set * L->n_ch; + const struct lid_hmx_set S = hctx->sets[set]; + if (c >= S.n_ch) { + continue; + } + + uint32_t n_q = 0; + if (gc != first) { + n_q = lid_hmx_push_q(hctx, dma_q, qs, &S, c) + n_in; + n_in = 0; + } + + // scales and G while the Q rows land + const uint32_t slot = lid_hmx_slot(hctx, hctx->qbuf, set); + lid_hmx_q_scg(hctx, slot, &S, c); + + for (uint32_t i = 0; i < n_q; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_q_tiles(hctx, slot, &S, c, qs, qh); + } + + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, (uint16_t) gc0); + + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(dma_q); + } + + lid_hmx_stage(hctx, ith, nth); +} + +// Run start when its Q was prepped in the previous run: block 0 K and mask, then stage -1 +static void lid_hmx_first_thread(unsigned int nth, unsigned int ith, void * data) { + const struct htp_lid_hmx_context * hctx = (const struct htp_lid_hmx_context *) data; + + const uint32_t n_in = lid_hmx_push_in(hctx, ith, nth, 0); + for (uint32_t i = 0; i < n_in; ++i) { + dma_queue_pop(hctx->octx->ctx->dma[ith]); + } + lid_hmx_stage(hctx, ith, nth); +} + +static int lid_hmx_run(struct htp_ops_context * octx, const struct htp_lid_kernel_params * kparams) { + const struct htp_tensor * q = octx->src[0]; + + const uint32_t D = q->ne[0]; + const uint32_t n_head = q->ne[1]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_threads = kparams->n_threads; + const uint32_t n_qsets = kparams->n_qsets; + + struct htp_lid_hmx_context hctx; + memset(&hctx, 0, sizeof(hctx)); + hctx.octx = octx; + hctx.kp = kparams; + hctx.Bc = kparams->kv_block; + hctx.kv_seg = kparams->kv_seg; + hctx.n_tile = kparams->n_tokens_tile; + hctx.n_tt = kparams->n_tt; + hctx.n_qsets = n_qsets; + htp_lid_hmx_vtcm_layout_build(&hctx.L, D, n_head, hctx.n_tile, hctx.Bc, n_qsets * kparams->n_qbuf, n_threads, + kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + const struct htp_lid_hmx_vtcm_layout * L = &hctx.L; + + hctx.qw = VTCM_LAYOUT_PTR(__fp16, base, L->off_qw); + hctx.gw = VTCM_LAYOUT_PTR(__fp16, base, L->off_g); + hctx.sc = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc); + hctx.sc_id = VTCM_LAYOUT_PTR(uint8_t, base, L->off_sc_id); + hctx.s = VTCM_LAYOUT_PTR(__fp16, base, L->off_s); + hctx.w = VTCM_LAYOUT_PTR(float, base, L->off_w); + hctx.qs = VTCM_LAYOUT_PTR(uint8_t, base, L->off_qs); + for (int i = 0; i < 2; i++) { + hctx.k_raw[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_k_raw[i]); + hctx.k_t[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_k_t[i]); + hctx.o[i] = VTCM_LAYOUT_PTR(__fp16, base, L->off_o[i]); + hctx.out[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_out[i]); + } + for (int i = 0; i < 4; i++) { + hctx.m[i] = VTCM_LAYOUT_PTR(uint8_t, base, L->off_m[i]); + } + + hmx_init_column_scales(hctx.sc_id, Q6_V_vsplat_R(0x3c00)); // identity 1.0 f16 + + hmx_queue_t hmx_q = octx->ctx->hmx_queue; + lid_hmx_job_t job[2]; + + const uint32_t n_tiles = n_stream * hctx.n_tt; + + for (uint32_t tile0 = 0, r = 0; tile0 < n_tiles; tile0 += n_qsets, ++r) { + hctx.tile0 = tile0; + hctx.n_run = MIN(n_qsets, n_tiles - tile0); + hctx.qbuf = r & (kparams->n_qbuf - 1); // n_qbuf is 1 or 2 + + for (uint32_t i = 0; i < hctx.n_run; ++i) { + hctx.sets[i] = lid_hmx_set_init(&hctx, i); + } + + // all sets of a run have the same token count: one set per run, or all tokens in one tile + const struct lid_hmx_set S0 = hctx.sets[0]; + hctx.nt_run = S0.nt; + hctx.n_u_run = S0.n_u; + hctx.n_blocks = hctx.n_run == n_qsets ? kparams->n_kv_blocks : (hctx.n_run * hctx.kv_seg + hctx.Bc - 1) / hctx.Bc; + + const int32_t n = (int32_t) hctx.n_blocks; + const uint32_t n_tot = hctx.n_run * L->n_ch; + const uint32_t per = fastdiv(n_tot + n_threads - 1, &octx->n_threads_div); + + hctx.n_qthr = (n_tot + per - 1) / per; + hctx.g = -1; + + const bool q_ready = hctx.nx_on; + if (q_ready) { + hctx.n_qthr = 0; + hctx.nx_on = false; + work_queue_run(octx->ctx->work_queue, lid_hmx_first_thread, &hctx, n_threads); + } else { + work_queue_run(octx->ctx->work_queue, lid_hmx_qprep_thread, &hctx, n_threads); + } + + // with two Q buffers the stages of this run prep the Q of the next one + if (kparams->n_qbuf == 2 && tile0 + n_qsets < n_tiles) { + struct htp_lid_hmx_context nx = hctx; + nx.tile0 = tile0 + n_qsets; + hctx.nx_n_run = MIN(n_qsets, n_tiles - nx.tile0); + for (uint32_t i = 0; i < hctx.nx_n_run; ++i) { + hctx.nx_sets[i] = lid_hmx_set_init(&nx, i); + } + hctx.nx_stg_div = init_fastdiv_values(hctx.n_blocks + 1); + hctx.nx_on = true; + } + + for (int32_t g = 0; g < n; ++g) { + struct lid_hmx_seg seg[LID_HMX_MAX_SEGS]; + uint32_t n_rows; + const uint32_t n_seg = lid_hmx_segs(&hctx, g, seg, &n_rows); + + lid_hmx_job_t * j = &job[g % 2]; + #pragma clang loop vectorize(disable) + for (uint32_t i = 0; i < n_seg; ++i) { + const uint32_t slot = lid_hmx_slot(&hctx, hctx.qbuf, seg[i].set); + j->seg[i].qw = hctx.qw + slot * L->n_ch * L->n_dot * HMX_FP16_TILE_N_ELMS; + j->seg[i].gw = hctx.gw + slot * L->n_ch * HMX_FP16_TILE_N_ELMS; + j->seg[i].sc = hctx.sc + slot * L->n_ch * 256; + j->seg[i].rt0 = seg[i].x0 / 32; + j->seg[i].n_rt = (seg[i].nv + 31) / 32; + } + j->n_seg = n_seg; + j->k_t = hctx.k_t[g % 2]; + j->sc_id = hctx.sc_id; + j->s = hctx.s; + j->o = hctx.o[g % 2]; + j->n_ch = S0.n_ch; + j->n_u = S0.n_u; + j->n_dot = L->n_dot; + j->n_head = n_head; + j->n_sg = L->n_sg; + hmx_queue_push(hmx_q, hmx_queue_make_desc(lid_hmx_worker, j)); + + // stage g has work only if there is a next block, a previous one or next run Q work + hctx.g = g; + if (g + 1 < n || g >= 1 || hctx.nx_on) { + work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_threads); + } + + hmx_queue_pop(hmx_q); + } + + // last stage only extracts; with one token only thread 0 has work (same token split for nth 1) + hctx.g = n; + const uint32_t n_last = hctx.nt_run == 1 && !hctx.nx_on ? 1 : n_threads; + work_queue_run(octx->ctx->work_queue, lid_hmx_stage_thread, &hctx, n_last); + } + + return HTP_STATUS_OK; +} + +int op_lightning_indexer(struct htp_ops_context * octx) { + const struct htp_tensor * q = octx->src[0]; + const struct htp_tensor * m = octx->src[3]; + + // no multi-device split yet, device 0 does all the work + if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx != 0) { + return HTP_STATUS_OK; + } + + const struct htp_lid_kernel_params * kparams = (const struct htp_lid_kernel_params *) octx->kernel_params; + + if (kparams->vtcm_size > octx->ctx->vtcm_size) { + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) { + return HTP_STATUS_INVAL_PARAMS; + } + + if (kparams->kernel_type == HTP_LID_KERNEL_HMX) { + return lid_hmx_run(octx, kparams); + } + + struct htp_lid_context lctx; + lctx.octx = octx; + lctx.kp = kparams; + lctx.n_blocks = kparams->n_kv_blocks; + lctx.blocks_per_thread = kparams->blocks_per_thread; + + htp_lid_vtcm_layout_build(&lctx.L, q->ne[0], q->ne[1], kparams->n_tokens_tile, kparams->n_threads, kparams->k_row_size); + + uint8_t * const base = octx->ctx->vtcm_base; + + lctx.spad_q = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_q); + lctx.spad_w = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_w); + lctx.spad_qs = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_qs); + lctx.spad_k = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_k); + lctx.spad_kt = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_kt); + lctx.spad_m = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_m); + lctx.spad_o = VTCM_LAYOUT_PTR(uint8_t, base, lctx.L.off_o); + + const uint32_t n_tokens = q->ne[2]; + const uint32_t n_stream = q->ne[3]; + const uint32_t n_tile = kparams->n_tokens_tile; + const uint32_t n_threads = kparams->n_threads; + + for (uint32_t is = 0; is < n_stream; ++is) { + lctx.is = is; + lctx.im3 = fastmodulo(is, m->ne[3], &kparams->div_m_ne3); + + for (uint32_t t0 = 0; t0 < n_tokens; t0 += n_tile) { + lctx.t0 = t0; + lctx.nt = MIN(n_tile, n_tokens - t0); + + work_queue_run(octx->ctx->work_queue, lid_prep_thread, &lctx, MIN(n_threads, lctx.nt * q->ne[1])); + work_queue_run(octx->ctx->work_queue, lid_compute_thread, &lctx, n_threads); + } + } + + return HTP_STATUS_OK; +} diff --git a/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h new file mode 100644 index 000000000000..6571db32b9d8 --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/lightning-indexer-ops.h @@ -0,0 +1,213 @@ +#ifndef HTP_LIGHTNING_INDEXER_OPS_H +#define HTP_LIGHTNING_INDEXER_OPS_H + +#include +#include +#include + +#include "hex-common.h" +#include "hex-fastdiv.h" +#include "htp-vtcm.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define HTP_LID_KV_BLOCK 64 // kv rows per block (HVX and HMX Bc) +#define HTP_LID_TOKENS_MAX 32 // max tokens per Q tile (HVX splat path) +#define HTP_LID_HMX_TOKENS_MAX 512 // HMX can keep more tokens; VTCM decides +#define HTP_LID_HMX_KV_BLOCK_MAX 2048 +#define HTP_LID_HMX_MAX_QSETS 8 // streams in one HMX pipeline run +#define HTP_LID_N_BUF 4 // K / mask / output buffers per thread (two blocks computed, two in flight) + +#ifndef HMX_FP16_TILE_N_COLS +#define HMX_FP16_TILE_N_COLS 32 +#endif +#ifndef HMX_FP16_TILE_SIZE +#define HMX_FP16_TILE_SIZE 2048 +#endif + +enum htp_lid_kernel_type { + HTP_LID_KERNEL_HVX = 0, + HTP_LID_KERNEL_HMX = 1, +}; + +struct htp_lid_kernel_params { + uint8_t kernel_type; // enum htp_lid_kernel_type + uint32_t n_threads; + uint32_t n_tokens_tile; // tokens per Q tile + uint32_t n_kv_blocks; // HVX: per stream, HMX: per run of n_qsets sets + uint32_t blocks_per_thread; + uint32_t vtcm_size; + uint32_t k_row_size; // K row bytes + uint32_t kv_block; // HMX: kv rows per block (multiple of 64) + uint32_t n_qsets; // HMX: Q sets kept in VTCM (streams run in one pipeline) + uint32_t n_qbuf; // HMX: Q buffers, 2: the next run's Q is prepped during the current run + uint32_t kv_seg; // HMX: rows per set, n_kv padded to 64 + uint32_t n_tt; // HMX: token tiles per stream + + struct fastdiv_values div_n_head; + struct fastdiv_values div_m_ne3; // mask streams (broadcast) + struct fastdiv_values div_kv_seg; // HMX + struct fastdiv_values div_n_tt; // HMX + struct fastdiv_values div_n_ch; // HMX: channel tiles per Q set +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#else +_Static_assert(sizeof(struct htp_lid_kernel_params) <= 128, "htp_lid_kernel_params is too large"); +#endif + +// VTCM layout shared by host (fit check) and device (buffer offsets). +// Shared regions: Q splats (one f16 vector per q element) and head weight splats for the current token tile. +// Per-thread regions: Q prep staging, K / mask / output buffers, two transposed K blocks and transpose scratch. +struct htp_lid_vtcm_layout { + size_t off_q; + size_t off_w; + size_t off_qs; + size_t off_k; + size_t off_kt; + size_t off_m; + size_t off_o; + + size_t size_q_row; // f16 Q row, padded + size_t size_qs_row; // f32 Q staging row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; // f16 mask row of one kv block, padded + size_t size_o_row; // f32 output row of one kv block, padded + + size_t size_qs; // per thread + size_t size_k_buf; // one K buffer + size_t size_kt; // per thread: two transposed K blocks, two 64x64 scratch tiles, Q8_0 block scales + size_t size_m_buf; // one mask buffer + size_t size_o_buf; // one output buffer + + size_t total_bytes; +}; + +static inline void htp_lid_vtcm_layout_build(struct htp_lid_vtcm_layout * L, size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) { + L->size_q_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(HTP_LID_KV_BLOCK * sizeof(float), 128); + + L->size_qs = 2 * (n_head * L->size_qs_row + hex_round_up(n_head * sizeof(float), 128)) + L->size_q_row; + L->size_k_buf = HTP_LID_KV_BLOCK * L->size_k_raw_row; + L->size_kt = (2 * hex_round_up(D, HTP_LID_KV_BLOCK) + 2 * HTP_LID_KV_BLOCK + D / 32) * 128; + L->size_m_buf = n_tok * L->size_m_row; + L->size_o_buf = n_tok * L->size_o_row; + + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_q, n_tok * n_head * D * 128); + VTCM_LAYOUT_ALLOC(off, off_w, n_tok * n_head * 128); + VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs); + VTCM_LAYOUT_ALLOC(off, off_k, n_threads * HTP_LID_N_BUF * L->size_k_buf); + VTCM_LAYOUT_ALLOC(off, off_kt, n_threads * L->size_kt); + VTCM_LAYOUT_ALLOC(off, off_m, n_threads * HTP_LID_N_BUF * L->size_m_buf); + VTCM_LAYOUT_ALLOC(off, off_o, n_threads * HTP_LID_N_BUF * L->size_o_buf); + + L->total_bytes = off; +} + +static inline size_t htp_lid_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t n_threads, size_t k_row_size) { + struct htp_lid_vtcm_layout L; + htp_lid_vtcm_layout_build(&L, D, n_head, n_tok, n_threads, k_row_size); + return L.total_bytes; +} + +// HMX path. K is the activation (kv rows), Q the weights (one output channel per (token, head)). +// mm1: S = relu(K Q^T) * |w| per channel (HMX convert scale + pos clamp) +// mm2: O = S G, G[(t,h), t] = sign(w[t,h]). O is [kv, token], f16. +// Q weights, G, scales and w are per Q set (one per stream when all tokens fit one tile). +struct htp_lid_hmx_vtcm_layout { + size_t off_qw; // Q weight tiles [n_qsets][n_ch][n_dot] + size_t off_g; // G weight tiles [n_qsets][n_ch] + size_t off_sc; // per channel scales, 256 bytes per channel tile, [n_qsets][n_ch] + size_t off_sc_id; // identity scales + size_t off_s; // S tiles of one block and token tile, [n_rt][n_sg] + size_t off_w; // w of the token tile, [n_qsets][n_tok][n_head] f32 + size_t off_qs; // per thread: f32 and f16 staging for 32 Q rows, gather scratch + size_t off_k_raw[2]; // K rows, [Bc][size_k_row] + size_t off_k_t[2]; // K tiles [n_rt][n_dot] + size_t off_o[2]; // O tiles [n_rt][n_u] + size_t off_m[4]; // mask rows [n_tok][size_m_row] + size_t off_out[2]; // f32 output rows [n_tok][size_o_row] + + size_t size_qs_row; // f32 Q row, padded + size_t size_qh_row; // f16 Q row, padded + size_t size_k_row; // f16 K row, padded + size_t size_k_raw_row; + size_t size_m_row; + size_t size_o_row; + size_t size_qs; // per thread + size_t size_w; // per Q set + + uint32_t n_ch; // channel tiles: ceil(n_tok * n_head / 32) + uint32_t n_u; // token tiles: ceil(n_tok / 32) + uint32_t n_dot; // D / 32 + uint32_t n_rt; // kv row tiles per block: Bc / 32 + uint32_t n_sg; // S tiles per (row tile, token tile): min(n_head, n_ch) + + size_t total_bytes; +}; + +static inline void htp_lid_hmx_vtcm_layout_build(struct htp_lid_hmx_vtcm_layout * L, + size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) { + L->n_ch = (uint32_t) ((n_tok * n_head + 31) / 32); + L->n_u = (uint32_t) ((n_tok + 31) / 32); + L->n_dot = (uint32_t) (D / 32); + L->n_rt = (uint32_t) (Bc / 32); + L->n_sg = (uint32_t) (n_head < L->n_ch ? n_head : L->n_ch); + + L->size_qs_row = hex_round_up(D * sizeof(float), 128); + L->size_qh_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_row = hex_round_up(D * sizeof(uint16_t), 128); + L->size_k_raw_row = L->size_k_row > hex_round_up(k_row_size, 128) ? L->size_k_row : hex_round_up(k_row_size, 128); + L->size_m_row = hex_round_up(Bc * sizeof(uint16_t), 128); + L->size_o_row = hex_round_up(Bc * sizeof(float), 128); + L->size_qs = 2 * 32 * L->size_qs_row + 32 * L->size_qh_row + hex_round_up(Bc / 64 * 128, 128); + L->size_w = hex_round_up(n_tok * n_head * sizeof(float), 128); + + const size_t tile = HMX_FP16_TILE_SIZE; + + // tile regions first: they need 2 KB alignment + size_t off = 0; + VTCM_LAYOUT_ALLOC(off, off_qw, n_qsets * L->n_ch * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_g, n_qsets * L->n_ch * tile); + VTCM_LAYOUT_ALLOC(off, off_s, (size_t) L->n_rt * L->n_sg * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[0], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_k_t[1], (size_t) L->n_rt * L->n_dot * tile); + VTCM_LAYOUT_ALLOC(off, off_o[0], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_o[1], (size_t) L->n_rt * L->n_u * tile); + VTCM_LAYOUT_ALLOC(off, off_sc, n_qsets * L->n_ch * 256); + VTCM_LAYOUT_ALLOC(off, off_sc_id, 256); + VTCM_LAYOUT_ALLOC(off, off_w, n_qsets * L->size_w); + VTCM_LAYOUT_ALLOC(off, off_qs, n_threads * L->size_qs); + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_k_raw[i], Bc * L->size_k_raw_row); + } + for (int i = 0; i < 4; i++) { + VTCM_LAYOUT_ALLOC(off, off_m[i], n_tok * L->size_m_row); + } + for (int i = 0; i < 2; i++) { + VTCM_LAYOUT_ALLOC(off, off_out[i], n_tok * L->size_o_row); + } + + L->total_bytes = off; +} + +static inline size_t htp_lid_hmx_compute_vtcm_usage(size_t D, size_t n_head, size_t n_tok, size_t Bc, size_t n_qsets, size_t n_threads, size_t k_row_size) { + struct htp_lid_hmx_vtcm_layout L; + htp_lid_hmx_vtcm_layout_build(&L, D, n_head, n_tok, Bc, n_qsets, n_threads, k_row_size); + return L.total_bytes; +} + +#ifdef __cplusplus +} +#endif + +#endif /* HTP_LIGHTNING_INDEXER_OPS_H */ diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 6a6d5a3a97ff..ac8820beb02d 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -1056,6 +1056,9 @@ static int execute_op(struct htp_ops_context * octx) { case HTP_OP_GATED_DELTA_NET: return op_gated_delta_net(octx); + case HTP_OP_LIGHTNING_INDEXER: + return op_lightning_indexer(octx); + case HTP_OP_TRI: return op_unary(octx); diff --git a/scripts/snapdragon/run.py b/scripts/snapdragon/run.py index bef6d4e50bcd..451e460865c4 100755 --- a/scripts/snapdragon/run.py +++ b/scripts/snapdragon/run.py @@ -33,6 +33,7 @@ "GGML_HEXAGON_MM_SELECT", "GGML_HEXAGON_FA_SELECT", "GGML_HEXAGON_FA_HEAD_SPLIT", + "GGML_HEXAGON_LI_SELECT", "GGML_HEXAGON_GDN_SELECT", "GGML_HEXAGON_AR_SELECT", "GGML_HEXAGON_AR_SCATTER", @@ -235,6 +236,7 @@ def main(): parser.add_argument("--hex-mm-select", help="Select MUL_MAT and MUL_MAT_ID kernel (GGML_HEXAGON_MM_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-fa-select", help="Select Flash Attention kernel (GGML_HEXAGON_FA_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-fa-head-split", help="Enable (1) or disable (0) head-parallel flash_attn partitioning (GGML_HEXAGON_FA_HEAD_SPLIT)") + parser.add_argument("--hex-li-select", help="Select Lightning Indexer kernel (GGML_HEXAGON_LI_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-gdn-select", help="Select Gated Delta Net kernel (GGML_HEXAGON_GDN_SELECT) 2:HMX,1:HVX,0:disable") parser.add_argument("--hex-ar-select", help="Select All-Reduce kernel (GGML_HEXAGON_AR_SELECT) 1:enable,0:disable") parser.add_argument("--hex-ar-scatter", help="Enable (1) or disable (0) reduce-scatter for fused ALLREDUCE+ADD (GGML_HEXAGON_AR_SCATTER)") @@ -388,6 +390,7 @@ def set_env(env_name, opt_val): set_env("GGML_HEXAGON_MM_SELECT", args.hex_mm_select) set_env("GGML_HEXAGON_FA_SELECT", args.hex_fa_select) set_env("GGML_HEXAGON_FA_HEAD_SPLIT", args.hex_fa_head_split) + set_env("GGML_HEXAGON_LI_SELECT", args.hex_li_select) set_env("GGML_HEXAGON_GDN_SELECT", args.hex_gdn_select) set_env("GGML_HEXAGON_AR_SELECT", args.hex_ar_select) set_env("GGML_HEXAGON_AR_SCATTER", args.hex_ar_scatter) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 1630ff52c138..f9cb12c3b975 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -12795,6 +12795,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_dsv4_hc_post(2560, n_tokens, true)); // synthetic, Qwen4-Exp kernel without the gate chain } + // Qwen3.8-Flash-Next pooled indexer prefill. + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_F32)); + test_cases.emplace_back(new test_lightning_indexer(128, 4, 1088, 1024, 1, 1, GGML_TYPE_Q4_0)); // SWIGLU at a 27B-class FFN width, fused [gate|up] vs split operands // note: same bytes either way, so a backend that indexes them differently shows it here for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) { From 4b29d02629b76160350ded907c64b733d98a54d1 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 09:22:54 +0200 Subject: [PATCH 038/286] common : fix -hf tag matching inside longer quant names (#31620) * common : fix -hf tag matching inside longer quant names ":F16" picked a BF16 file when the repo had both, since "BF16" contains "F16". The same applied to ":Q4_XS" vs "IQ4_XS". The tag now has to start at a token boundary in the path. Assisted-by: dsh:llama.cpp/ggml-org/GLM-5.3-Flash-GGUF (Mac Studio, M5 Ultra, 256GB) * common : accept underscore as -hf tag boundary Some repos separate the quant with an underscore, e.g. stepfun-ai/Step-3.5-Flash-GGUF-Q4_K_S ships step3p5_flash_Q4_K_S-00001-of-00012.gguf, which :Q4_K_S no longer matched. Add _ to the token boundary and pin the tag and default lookup cases. --------- Co-authored-by: Pascal --- common/download.cpp | 3 +- tests/test-model-resolution.cpp | 88 ++++++++++++++++++++++++++------- 2 files changed, 73 insertions(+), 18 deletions(-) diff --git a/common/download.cpp b/common/download.cpp index 08531a2d09d4..4de93f30defa 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -748,7 +748,8 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, } for (const auto & t : tags) { - std::regex pattern(t + "[.-]", std::regex::icase); + // match the tag as a full token, so ":F16" does not match a BF16 file + std::regex pattern("(^|[-./_])" + t + "[.-]", std::regex::icase); for (const auto & f : files) { if (gguf_filename_is_model(f.path) && std::regex_search(f.path, pattern)) { diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 705fd65fa950..73a87482d2c1 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -153,6 +153,20 @@ static const std::vector vendors = { "BlackSheep-24B.Q8_0.gguf", }; +// a tag must match the full token: F16 ends inside BF16, Q4_XS inside IQ4_XS +static const std::vector twins = { + "model-BF16.gguf", + "model-F16.gguf", + "model-IQ4_XS.gguf", + "model-Q4_XS.gguf", +}; + +// an underscore separates the quant like a dash, as in step3p5_flash_Q4_K_S.gguf +static const std::vector underscore = { + "model_Q4_K_S.gguf", + "model_Q8_0.gguf", +}; + // every speculative sidecar type at the same quant static const std::vector quad = { "model-Q8_0.gguf", @@ -270,6 +284,32 @@ static const plan_case plan_cases[] = { "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"}, "", "", "", "", ""}, + // the tag picks the file carrying it as a full token, never a longer one + {"twins f16 tag", twins, "test/repo:F16", "", false, false, + "model-F16.gguf", {"model-F16.gguf"}, + "", "", "", "", ""}, + + {"twins bf16 tag", twins, "test/repo:BF16", "", false, false, + "model-BF16.gguf", {"model-BF16.gguf"}, + "", "", "", "", ""}, + + {"twins q4_xs tag", twins, "test/repo:Q4_XS", "", false, false, + "model-Q4_XS.gguf", {"model-Q4_XS.gguf"}, + "", "", "", "", ""}, + + {"twins iq4_xs tag", twins, "test/repo:IQ4_XS", "", false, false, + "model-IQ4_XS.gguf", {"model-IQ4_XS.gguf"}, + "", "", "", "", ""}, + + // the token boundary accepts an underscore, for the tag and the default lookup + {"underscore tag", underscore, "test/repo:Q4_K_S", "", false, false, + "model_Q4_K_S.gguf", {"model_Q4_K_S.gguf"}, + "", "", "", "", ""}, + + {"underscore default", underscore, "test/repo", "", false, false, + "model_Q8_0.gguf", {"model_Q8_0.gguf"}, + "", "", "", "", ""}, + // every sidecar type resolves at the tag {"quad exact tag", quad, "test/repo:Q8_0", "", true, false, "model-Q8_0.gguf", {"model-Q8_0.gguf"}, @@ -868,7 +908,7 @@ static const real_repo real_repos[] = { {"Q8_0", "MTP/Qwen3.8-27B-Uncensored-OrcaRouter-MTP-Q8_0.gguf", 1, "", "", "", "", ""}, {"BF16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}, {"Q5_K_M", "Qwen3.8-27B-Uncensored-OrcaRouter-Q5_K_M.gguf", 1, "", "", "", "", ""}, - {"F16", "BF16/Qwen3.8-27B-Uncensored-OrcaRouter-FP8-derived-BF16-00001-of-00002.gguf", 2, "", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, {"ChrisColeTech/qwen-image-edit-uncensored-GGUF", @@ -980,7 +1020,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-MTP-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"IQ2_M", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-NEO-MAX-IQ2_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-9B-The-Defiant-Fable-Uncnr-Heretic-plusIQ-NEO-MAX-MTP-bf16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "BF16", "Q4_K_S", "Q4_K_M", "IQ2_M"}}, {"DavidAU/Qwen3.6-27B-Heretic-Uncensored-FINETUNE-NEO-CODE-Di-IMatrix-MAX-GGUF", @@ -1044,7 +1084,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwythos-27B-MTP-Q4_K_M.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, {"Q8_0", "Qwythos-27B-MTP-Q8_0.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, {"BF16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}, - {"F16", "Qwythos-27B-BF16.gguf", 1, "mmproj-Qwythos-27B-F16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "BF16"}}, {"empero-ai/Qwythos-9B-v2-GGUF", @@ -1199,6 +1239,20 @@ static const real_repo real_repos[] = { {"BF16", "", 0, "", "", "", "eagle3-gpt-oss-120b-BF16.gguf", ""}}, {"MXFP4"}}, + // F16 and BF16 in one repo, the F16 tag must not pick the BF16 file + {"ggml-org/Llama-2-7B-GGUF", + {{"Llama-2-7B-BF16.gguf"}, + {"Llama-2-7B-F16.gguf"}, + {"Llama-2-7B-Q4_0.gguf"}, + {"Llama-2-7B-Q8_0.gguf"}}, + { + {"", "Llama-2-7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q8_0", "Llama-2-7B-Q8_0.gguf", 1, "", "", "", "", ""}, + {"Q4_0", "Llama-2-7B-Q4_0.gguf", 1, "", "", "", "", ""}, + {"F16", "Llama-2-7B-F16.gguf", 1, "", "", "", "", ""}, + {"BF16", "Llama-2-7B-BF16.gguf", 1, "", "", "", "", ""}}, + {"F16", "Q4_0", "Q8_0", "BF16"}}, + {"ggml-org/MiMo-V2.6-Distill-Qwen-9B-GGUF", {{"MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}, {"mmproj-MiMo-V2.6-Distill-Qwen-9B-Q8_0.gguf"}}, @@ -1412,7 +1466,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"Q8_0", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-27B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q4_K_M", "Q3_K_M", "BF16"}}, {"HauhauCS/Qwen3.5-2B-Uncensored-HauhauCS-Aggressive", @@ -1426,7 +1480,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"Q8_0", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, {"BF16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}, - {"F16", "Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-BF16.gguf", 1, "mmproj-Qwen3.5-2B-Uncensored-HauhauCS-Aggressive-f16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"HauhauCS/Qwen3.5-4B-Uncensored-HauhauCS-Aggressive", @@ -1622,7 +1676,7 @@ static const real_repo real_repos[] = { {"IQ2_M", "Qwen3.8-27B-Uncensored-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, {"Q4_0", "Qwen3.8-27B-Uncensored-draft-Q4_0.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}, {"F16", "", 0, "", "", "", "", ""}, - {"mtp", "Qwen3.8-27B-Uncensored-noMTP-IQ2_M.gguf", 1, "mmproj-Qwen3.8-27B-Uncensored-F16.gguf", "", "", "", ""}}, + {"mtp", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q5_K_M", "Q4_K_M", "IQ2_M", "Q4_0"}}, {"kai-os/Carnice-V2-27b-GGUF", @@ -1903,7 +1957,7 @@ static const real_repo real_repos[] = { {"Q4_K_M", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q4_K_M.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"Q8_0", "Huihui-Qwen3-VL-4B-Instruct-abliterated-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "Huihui-Qwen3-VL-4B-Instruct-abliterated-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_0", "Q4_K_M", "IQ4_XS", "BF16"}}, {"OBLITERATUS/Gemma-4-12B-OBLITERATED", @@ -2291,7 +2345,7 @@ static const real_repo real_repos[] = { {"Q8_0", "embeddinggemma-2-Q8_0.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, {"BF16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q6_K_XL", "embeddinggemma-2-UD-Q6_K_XL.gguf", 1, "mmproj-Q8_0.gguf", "", "", "", ""}, - {"F16", "embeddinggemma-2-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q6_K_XL", "Q4_K_XL", "Q8_0", "BF16"}}, {"unsloth/GLM-4.7-Flash-GGUF", @@ -2320,7 +2374,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/GLM-5.3-Flash-Q8_0-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q6_K_XL", "UD-Q6_K_XL/GLM-5.3-Flash-UD-Q6_K_XL-00001-of-00007.gguf", 7, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/GLM-5.3-Flash-BF16-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q6_K_XL", "IQ4_XS", "Q8_0", "BF16"}}, {"unsloth/GLM-5.3-GGUF", @@ -2359,7 +2413,7 @@ static const real_repo real_repos[] = { {"", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Kimi-K2.6-UD-Q8_K_XL-00001-of-00014.gguf", 14, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Kimi-K2.6-BF16-00001-of-00046.gguf", 46, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q4_K_XL", "BF16"}}, {"unsloth/Kimi-K2.7-Code-GGUF", @@ -2405,7 +2459,7 @@ static const real_repo real_repos[] = { {"Q8_0", "LFM2.5-VL-3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "LFM2.5-VL-3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "LFM2.5-VL-3B-BF16.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q8_0", "Q5_K_S", "Q4_K_M", "BF16"}}, {"unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", @@ -2449,7 +2503,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3-VL-30B-A3B-Instruct-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-TQ1_0", "Qwen3-VL-30B-A3B-Instruct-UD-TQ1_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3-VL-30B-A3B-Instruct-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"TQ1_0", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-27B-GGUF", @@ -2467,7 +2521,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-27B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.5-27B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-27B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q8_0", "Q6_K", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-35B-A3B-GGUF", @@ -2485,7 +2539,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.5-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.5-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "IQ3_S", "Q8_0", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.5-397B-A17B-GGUF", @@ -2503,7 +2557,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/Qwen3.5-397B-A17B-Q8_0-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Qwen3.5-397B-A17B-UD-Q8_K_XL-00001-of-00010.gguf", 10, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.5-397B-A17B-BF16-00001-of-00017.gguf", 17, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "IQ3_XXS", "Q8_0", "Q4_K_M", "BF16"}}, {"unsloth/Qwen3.6-35B-A3B-MTP-GGUF", @@ -2519,7 +2573,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Qwen3.6-35B-A3B-Q8_0.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf", 1, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Qwen3.6-35B-A3B-BF16-00001-of-00002.gguf", 2, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, {"unsloth/Qwen3.8-Flash-Next-GGUF", @@ -2557,7 +2611,7 @@ static const real_repo real_repos[] = { {"Q8_0", "Q8_0/Step-3.7-Flash-Q8_0-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, {"BF16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}, {"UD-Q8_K_XL", "UD-Q8_K_XL/Step-3.7-Flash-UD-Q8_K_XL-00001-of-00006.gguf", 6, "mmproj-BF16.gguf", "", "", "", ""}, - {"F16", "BF16/Step-3.7-Flash-BF16-00001-of-00008.gguf", 8, "mmproj-BF16.gguf", "", "", "", ""}}, + {"F16", "", 0, "", "", "", "", ""}}, {"Q8_K_XL", "Q3_K_M", "Q8_0", "BF16"}}, {"vmlinux/Qwen3.8-Flash-Next-Uncensored-Gufo-Q4Mix-GGUF", From 960b1d51b73176ac970be51ef8ef22c9e2b9113c Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 09:29:26 +0200 Subject: [PATCH 039/286] ggml : write host-buffer tensor data directly in gguf writer (#31144) The gguf writer staged each tensor into a temporary vector before writing it. For host-buffer tensors the data pointer is directly writable, so skip the staging copy and write it straight to the file. This avoids the per-tensor allocation, zero-init and memcpy. Assisted-by: dsh:llama.cpp/ggml-org/DeepSeek-V4-Flash-Vision-Exp-GGUF (Mac Studio, M2 Ultra, 192GB) --- ggml/src/gguf.cpp | 50 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 75ceade7d5fb..c80b682185ce 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -1463,6 +1463,7 @@ struct gguf_writer_base { // we bet on devirtualization virtual void write(int8_t val) = 0; virtual void write(const std::vector & val) = 0; + virtual void write(const void * data, size_t size) = 0; virtual void write_tensor_data(const struct gguf_tensor_info & info, size_t offset_data, size_t alignment) = 0; template @@ -1579,21 +1580,31 @@ struct gguf_writer_buf final : public gguf_writer_base { written_bytes += val.size(); } + void write(const void * data, const size_t size) override { + buf.insert(buf.end(), (const int8_t *) data, (const int8_t *) data + size); + written_bytes += size; + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(buf.size() - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); - const size_t offset = buf.size(); const size_t nbytes = ggml_nbytes(&info.t); - buf.resize(offset + nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data() + offset, info.t.data, nbytes); + const size_t offset = buf.size(); + buf.resize(offset + nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data() + offset, info.t.data, nbytes); + } + written_bytes += nbytes; } - written_bytes += nbytes; pad(alignment); } @@ -1624,20 +1635,33 @@ struct gguf_writer_file final : public gguf_writer_base { } } + void write(const void * data, const size_t size) override { + const auto ret = fwrite(data, 1, size, file); + written_bytes += size; + if (ret != size) { + throw std::runtime_error("unexpected fwrite number of bytes written, '" + std::to_string(ret) + "' instead of '" + std::to_string(size) + "'"); + } + } + void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override { GGML_ASSERT(written_bytes - offset_data == info.offset); GGML_ASSERT(ggml_is_contiguous(&info.t)); const size_t nbytes = ggml_nbytes(&info.t); - std::vector buf(nbytes); - if (info.t.buffer) { - ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + if (info.t.buffer && ggml_backend_buffer_is_host(info.t.buffer)) { + GGML_ASSERT(info.t.data != NULL && "tensor not allocated"); + write(info.t.data, nbytes); } else { - GGML_ASSERT(info.t.data); - memcpy(buf.data(), info.t.data, nbytes); + std::vector buf(nbytes); + if (info.t.buffer) { + ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes); + } else { + GGML_ASSERT(info.t.data); + memcpy(buf.data(), info.t.data, nbytes); + } + write(buf); } - write(buf); pad(alignment); } From bbb02c93e9b83255b70ac1a755bb02b0a251f9a7 Mon Sep 17 00:00:00 2001 From: Pascal Date: Wed, 4 Nov 2026 08:47:11 +0100 Subject: [PATCH 040/286] ggml: bound views by the bytes their strides reach (#31584) * ggml: bound views by the bytes their strides reach A view was checked against its source with the size of a contiguous tensor of the same shape, computed before its strides were set. Views whose rows overlap failed that check although they stay inside their source, and views with wider strides passed it while reaching past the end. The view functions now pass their strides to the tensor constructor, which checks the bytes they actually reach, like ggml_nbytes. The conv windows of delta_net are created with all their rows at once instead of being widened after the check. * ggml: assert that a strided view starts from contiguous elements The view functions set nb[0] to the type size, so the source must have contiguous elements along dim 0 for the view to read them. --- ggml/src/ggml.c | 71 ++++++++++++++++++----------------- src/models/delta-net-base.cpp | 5 +-- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index b91d98845826..8183525aff92 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1788,6 +1788,7 @@ static struct ggml_tensor * ggml_new_tensor_impl( enum ggml_type type, int n_dims, const int64_t * ne, + const size_t * nb, struct ggml_tensor * view_src, size_t view_offs) { @@ -1814,8 +1815,6 @@ static struct ggml_tensor * ggml_new_tensor_impl( data_size *= ne[i]; } - GGML_ASSERT(view_src == NULL || data_size == 0 || data_size + view_offs <= ggml_nbytes(view_src)); - void * data = view_src != NULL ? view_src->data : NULL; if (data != NULL) { data = (char *) data + view_offs; @@ -1859,12 +1858,23 @@ static struct ggml_tensor * ggml_new_tensor_impl( result->ne[i] = ne[i]; } - result->nb[0] = ggml_type_size(type); - result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); - for (int i = 2; i < GGML_MAX_DIMS; i++) { - result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + // a view built with strides takes them from the caller, any other tensor is contiguous + if (nb != NULL) { + for (int i = 0; i < GGML_MAX_DIMS; i++) { + result->nb[i] = nb[i]; + } + } else { + result->nb[0] = ggml_type_size(type); + result->nb[1] = result->nb[0]*(result->ne[0]/ggml_blck_size(type)); + for (int i = 2; i < GGML_MAX_DIMS; i++) { + result->nb[i] = result->nb[i - 1]*result->ne[i - 1]; + } } + // a view stays inside its source, over the bytes its strides actually reach + const size_t view_size = ggml_nbytes(result); + GGML_ASSERT(view_src == NULL || view_size == 0 || view_size + view_offs <= ggml_nbytes(view_src)); + ctx->n_objects++; return result; @@ -1875,7 +1885,7 @@ struct ggml_tensor * ggml_new_tensor( enum ggml_type type, int n_dims, const int64_t * ne) { - return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, 0); + return ggml_new_tensor_impl(ctx, type, n_dims, ne, NULL, NULL, 0); } struct ggml_tensor * ggml_new_tensor_1d( @@ -1992,13 +2002,9 @@ struct ggml_tensor * ggml_format_name(struct ggml_tensor * tensor, const char * struct ggml_tensor * ggml_view_tensor( struct ggml_context * ctx, struct ggml_tensor * src) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, src->type, GGML_MAX_DIMS, src->ne, src->nb, src, 0); ggml_format_name(result, "%s (view)", src->name); - for (int i = 0; i < GGML_MAX_DIMS; i++) { - result->nb[i] = src->nb[i]; - } - return result; } @@ -2948,7 +2954,7 @@ static struct ggml_tensor * ggml_glu_impl( } int64_t ne[GGML_MAX_DIMS] = { a->ne[0] / 2 }; for (int i = 1; i < GGML_MAX_DIMS; i++) ne[i] = a->ne[i]; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b ? a->ne : ne, NULL, NULL, 0); ggml_set_op_params_i32(result, 0, (int32_t) op); ggml_set_op_params_i32(result, 1, (int32_t) swapped); @@ -3757,7 +3763,7 @@ struct ggml_tensor * ggml_reshape( // as only the shape of b is relevant, and not its memory layout, b is allowed to be non contiguous. GGML_ASSERT(ggml_nelements(a) == ggml_nelements(b)); - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, GGML_MAX_DIMS, b->ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3774,7 +3780,7 @@ struct ggml_tensor * ggml_reshape_1d( GGML_ASSERT(ggml_nelements(a) == ne0); const int64_t ne[1] = { ne0 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 1, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3792,7 +3798,7 @@ struct ggml_tensor * ggml_reshape_2d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1); const int64_t ne[2] = { ne0, ne1 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 2, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3811,7 +3817,7 @@ struct ggml_tensor * ggml_reshape_3d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2); const int64_t ne[3] = { ne0, ne1, ne2 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 3, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3831,7 +3837,7 @@ struct ggml_tensor * ggml_reshape_4d( GGML_ASSERT(ggml_nelements(a) == ne0*ne1*ne2*ne3); const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, a, 0); + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, 4, ne, NULL, a, 0); ggml_format_name(result, "%s (reshaped)", a->name); result->op = GGML_OP_RESHAPE; @@ -3845,8 +3851,12 @@ static struct ggml_tensor * ggml_view_impl( struct ggml_tensor * a, int n_dims, const int64_t * ne, + const size_t * nb, size_t offset) { - struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, a, offset); + // a strided view counts its strides from elements that are contiguous along dim 0 of its source + GGML_ASSERT(nb == NULL || a->nb[0] == ggml_type_size(a->type)); + + struct ggml_tensor * result = ggml_new_tensor_impl(ctx, a->type, n_dims, ne, nb, a, offset); ggml_format_name(result, "%s (view)", a->name); ggml_set_op_params(result, &offset, sizeof(offset)); @@ -3864,7 +3874,7 @@ struct ggml_tensor * ggml_view_1d( struct ggml_tensor * a, int64_t ne0, size_t offset) { - struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, offset); + struct ggml_tensor * result = ggml_view_impl(ctx, a, 1, &ne0, NULL, offset); return result; } @@ -3879,12 +3889,9 @@ struct ggml_tensor * ggml_view_2d( size_t nb1, size_t offset) { const int64_t ne[2] = { ne0, ne1 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb1*ne1, nb1*ne1 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = result->nb[1]*ne1; - result->nb[3] = result->nb[2]; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 2, ne, nb, offset); return result; } @@ -3901,12 +3908,9 @@ struct ggml_tensor * ggml_view_3d( size_t nb2, size_t offset) { const int64_t ne[3] = { ne0, ne1, ne2 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb2*ne2 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = result->nb[2]*ne2; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 3, ne, nb, offset); return result; } @@ -3925,12 +3929,9 @@ struct ggml_tensor * ggml_view_4d( size_t nb3, size_t offset) { const int64_t ne[4] = { ne0, ne1, ne2, ne3 }; + const size_t nb[4] = { ggml_type_size(a->type), nb1, nb2, nb3 }; - struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, offset); - - result->nb[1] = nb1; - result->nb[2] = nb2; - result->nb[3] = nb3; + struct ggml_tensor * result = ggml_view_impl(ctx, a, 4, ne, nb, offset); return result; } diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index 991b55ffb708..0a44db66f0b3 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -509,12 +509,11 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state( // window w starts at column n - (Kw - 1) + w (ascending time) and is rollback slot Kw - 1 - w // slots reaching back past this ubatch are left unwritten, like the ssm snapshots in build_recurrent_attn - // the windows overlap, so the view is created with one window and widened after the size check + // consecutive windows start one element apart, so they overlap ggml_tensor * windows = ggml_view_4d(ctx0, conv_input, - conv_kernel_size - 1, 1, conv_channels, n_seqs, + conv_kernel_size - 1, Kw, conv_channels, n_seqs, ggml_element_size(conv_input), conv_input->nb[1], conv_input->nb[2], ggml_row_size(conv_input->type, n - (Kw - 1))); - windows->ne[1] = Kw; // [d-1, slot, channel, seq] view of the cells of this ubatch ggml_tensor * slots = ggml_view_4d(ctx0, conv_states_all, From 6900fb5ac8344afd4103b136cfd631ca6707e038 Mon Sep 17 00:00:00 2001 From: lhez Date: Wed, 4 Nov 2026 00:07:14 -0800 Subject: [PATCH 041/286] opencl: improve q2_0 GEMV for Adreno (#31577) * opencl: improve q2_0 decoding and small f32 matmuls Assisted-by: Codex * opencl: optimize small f32 matmuls on Adreno 8xx Assisted-by: Codex --- ggml/src/ggml-opencl/ggml-opencl.cpp | 67 ++++++++++++++----- .../kernels/gemv_noshuffle_q2_0_f32.cl | 63 ++++++++++------- .../src/ggml-opencl/kernels/mul_mv_f32_f32.cl | 67 +++++++++++++++++++ 3 files changed, 157 insertions(+), 40 deletions(-) diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cpp index f0d401d55d22..22cf6f67db7d 100644 --- a/ggml/src/ggml-opencl/ggml-opencl.cpp +++ b/ggml/src/ggml-opencl/ggml-opencl.cpp @@ -887,6 +887,7 @@ struct ggml_backend_opencl_context { cl_kernel kernel_cpy_f32_f32_pack = nullptr; cl_kernel kernel_cpy_f32_f32_flat = nullptr; cl_kernel kernel_mul_mat_f32_f32; + cl_kernel kernel_mul_mv_f32_f32_r2 = nullptr; cl_kernel kernel_fwht[2] = {}; size_t fwht_workgroup_size[2] = {}; cl_kernel kernel_mul_mat_f16_f16; @@ -1027,10 +1028,13 @@ struct ggml_backend_opencl_context { cl_kernel kernel_mul_mv_q2_0_f32_flat; cl_kernel kernel_mul_mm_q2_0_f32_l4_lm; cl_kernel kernel_gemv_noshuffle_q2_0_f32 = nullptr; + cl_kernel kernel_gemv_noshuffle_q2_0_f32_wide = nullptr; + size_t q2_0_mv_workgroup_size = 512; + size_t q2_0_mv_workgroup_size_wide = 512; cl_kernel kernel_gemm_noshuffle_q2_0_f32 = nullptr; cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a = nullptr; cl_kernel kernel_gemm_noshuffle_q2_0_q8_1_dp4a_narrow = nullptr; - int q2_0_mv_nsg = 8; + int q2_0_mv_nsg = 32; cl_kernel kernel_mul_mv_q2_0_f32; cl_kernel kernel_convert_block_iq1_m, kernel_restore_block_iq1_m; cl_kernel kernel_mul_mv_iq1_m_f32_flat; @@ -3591,6 +3595,17 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { build_program_from_source(backend_ctx, kernel_src.c_str(), compile_opts); CL_CHECK((backend_ctx->kernel_mul_mat_f32_f32 = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mat_f32_f32", &err), err)); + if (backend_ctx->adreno_gen == ADRENO_GPU_GEN::A8X) { + cl_kernel kernel = clCreateKernel(backend_ctx->program_mul_mv_f32_f32, "kernel_mul_mv_f32_f32_r2", &err); + CL_CHECK(err); + size_t max_workgroup; + CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); + if (max_workgroup >= 64) { + backend_ctx->kernel_mul_mv_f32_f32_r2 = kernel; + } else { + CL_CHECK(clReleaseKernel(kernel)); + } + } GGML_LOG_CONT("."); } @@ -5076,25 +5091,30 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) { #endif if (const char * env = getenv("GGML_OPENCL_Q2_0_MV_NSG")) { const int nsg = atoi(env); - if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8) { + if (nsg == 1 || nsg == 2 || nsg == 4 || nsg == 8 || nsg == 16 || nsg == 32) { backend_ctx->q2_0_mv_nsg = nsg; } } - for (;;) { - const std::string opts = compile_opts + " -DQ2_0_MV_NSG=" + std::to_string(backend_ctx->q2_0_mv_nsg); + for (int wide = 0; wide < 2; ++wide) { + const std::string opts = compile_opts + " -DQ2_0_MV_WIDE=" + std::to_string(wide); cl_program prog = build_program_from_source(backend_ctx, kernel_src.c_str(), opts); cl_kernel kernel = clCreateKernel(prog, "kernel_gemv_noshuffle_q2_0_f32", &err); CL_CHECK(err); CL_CHECK(clReleaseProgram(prog)); size_t max_workgroup; CL_CHECK(clGetKernelWorkGroupInfo(kernel, backend_ctx->device, CL_KERNEL_WORK_GROUP_SIZE, sizeof(max_workgroup), &max_workgroup, nullptr)); - if (64*(size_t)backend_ctx->q2_0_mv_nsg <= max_workgroup) { + size_t workgroup = 512; + while (workgroup > max_workgroup) { + workgroup /= 2; + } + GGML_ASSERT(workgroup >= 64); + if (wide) { + backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide = kernel; + backend_ctx->q2_0_mv_workgroup_size_wide = workgroup; + } else { backend_ctx->kernel_gemv_noshuffle_q2_0_f32 = kernel; - break; + backend_ctx->q2_0_mv_workgroup_size = workgroup; } - CL_CHECK(clReleaseKernel(kernel)); - GGML_ASSERT(backend_ctx->q2_0_mv_nsg > 1); - backend_ctx->q2_0_mv_nsg /= 2; } GGML_LOG_CONT("."); } @@ -24248,7 +24268,15 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t const int k = src0->ne[0]; if (n == 1 || !ggml_is_contiguous(src1) || offset1 % backend_ctx->alignment != 0) { - cl_kernel gemv = backend_ctx->kernel_gemv_noshuffle_q2_0_f32; + const bool wide = m >= 16384 && m % 8 == 0; + cl_kernel gemv = wide ? backend_ctx->kernel_gemv_noshuffle_q2_0_f32_wide : backend_ctx->kernel_gemv_noshuffle_q2_0_f32; + const size_t workgroup = wide ? backend_ctx->q2_0_mv_workgroup_size_wide : backend_ctx->q2_0_mv_workgroup_size; + const size_t rows = wide ? 8 : 4; + size_t nsg = std::min((size_t)backend_ctx->q2_0_mv_nsg, wide ? size_t(8) : m >= 4096 ? size_t(16) : size_t(32)); + while (nsg > (size_t)k/64 || nsg*16 > workgroup) { + nsg /= 2; + } + const size_t lanes = workgroup/nsg; const int stride_d = dst->ne[0]; const int stride_b = src1->nb[1]/sizeof(float); CL_CHECK(clSetKernelArg(gemv, 0, sizeof(cl_mem), &extra0->qs)); @@ -24261,9 +24289,8 @@ static void ggml_cl_mul_mat_q2_0_f32_adreno(ggml_backend_t backend, const ggml_t CL_CHECK(clSetKernelArg(gemv, 7, sizeof(int), &m)); CL_CHECK(clSetKernelArg(gemv, 8, sizeof(int), &stride_b)); CL_CHECK(clSetKernelArg(gemv, 9, sizeof(int), &stride_d)); - const size_t nsg = backend_ctx->q2_0_mv_nsg; - size_t global[] = { (size_t)CEIL_DIV(m, 128)*64, (size_t)n*nsg, 1 }; - size_t local[] = { 64, nsg, 1 }; + size_t global[] = { (size_t)CEIL_DIV(m, rows*lanes)*lanes, (size_t)n*nsg, 1 }; + size_t local[] = { lanes, nsg, 1 }; backend_ctx->enqueue_ndrange_kernel(gemv, 3, global, local, dst); return; } @@ -30881,9 +30908,15 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co // Override with GGML_OPENCL_A7X_F32_LM_BYPASS=0. static const char * a7x_f32lm_env = getenv("GGML_OPENCL_A7X_F32_LM_BYPASS"); static const bool a7x_f32lm_bypass = (a7x_f32lm_env == nullptr || a7x_f32lm_env[0] != '0'); + // The 64-column tile wastes work on small F32 matrices, including block rotations. + const bool f32_use_mv = (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || + (backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr && ne00 <= 512)) && + src0t == GGML_TYPE_F32 && ne00 >= 256 && ne00 <= 1024 && ne00 % 16 == 0 && ne01 <= 512 && ne11 <= 16 && + ne02 == 1 && ne12 == 1 && ne13 == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && (offset0 | offset1) % 16 == 0; if (src1t == GGML_TYPE_F32 && ne00 % 16 == 0 && - ne11 > 1 && + ne11 > 1 && !f32_use_mv && !(a7x_f32lm_bypass && src0t == GGML_TYPE_F32 && ne11 > 8 && backend_ctx->adreno_gen == ADRENO_GPU_GEN::A7X)) { switch(src0t) { @@ -32072,6 +32105,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co GGML_ASSERT(src1t == GGML_TYPE_F32); kernel = backend_ctx->kernel_mul_mat_f32_f32; nrows = 4; + if (f32_use_mv && ne11 > 1 && backend_ctx->kernel_mul_mv_f32_f32_r2 != nullptr) { + kernel = backend_ctx->kernel_mul_mv_f32_f32_r2; + } if (backend_ctx->gpu_family == INTEL) { nth0 = 32; @@ -33833,8 +33869,9 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); } else { int64_t ny = (ne11 + nrows - 1)/nrows; + int64_t nx = kernel == backend_ctx->kernel_mul_mv_f32_f32_r2 ? CEIL_DIV(ne01, 2) : ne01; - size_t global_work_size[] = {(size_t)ne01*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; + size_t global_work_size[] = {(size_t)nx*nth0, (size_t)ny*nth1, (size_t)ne12*ne13}; size_t local_work_size[] = {(size_t)nth0, (size_t)nth1, 1}; backend_ctx->enqueue_ndrange_kernel(kernel, 3, global_work_size, local_work_size, dst); diff --git a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl index 27ff7edd34fc..a46293c248aa 100644 --- a/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl +++ b/ggml/src/ggml-opencl/kernels/gemv_noshuffle_q2_0_f32.cl @@ -1,7 +1,19 @@ #pragma OPENCL EXTENSION cl_khr_fp16 : enable -#ifndef Q2_0_MV_NSG -#define Q2_0_MV_NSG 8 +#if Q2_0_MV_WIDE +#define Q2_0_MV_ROWS 8 +#define Q2_0_MV_LOAD_D vload8 +#define Q2_0_MV_STORE vstore8 +#define Q2_0_MV_FLOAT convert_float8 +typedef float8 q2_0_mv_acc_t; +typedef uchar8 q2_0_mv_quant_t; +#else +#define Q2_0_MV_ROWS 4 +#define Q2_0_MV_LOAD_D vload4 +#define Q2_0_MV_STORE vstore4 +#define Q2_0_MV_FLOAT convert_float4 +typedef float4 q2_0_mv_acc_t; +typedef uchar4 q2_0_mv_quant_t; #endif kernel void kernel_gemv_noshuffle_q2_0_f32( @@ -19,46 +31,47 @@ kernel void kernel_gemv_noshuffle_q2_0_f32( src1 = (global const float *)((global const char *)src1 + offset1); dst = (global float *)((global char *)dst + offsetd); - const uint m = (uint)ne01; - const uint mh = m >> 1; + const uint mr = (uint)ne01/Q2_0_MV_ROWS; const uint lid = get_local_id(0); const uint sgi = get_local_id(1); + const uint lanes = get_local_size(0); + const uint nsg = get_local_size(1); const uint col = get_group_id(1); - const uint j = get_group_id(0)*64u + lid; - const uint row = j << 1; + const uint j = get_group_id(0)*lanes + lid; global const float * y = src1 + (ulong)col * (uint)ne10; - float2 sum = 0.0f; - if (j < mh) { - global const ushort * qs = (global const ushort *)src0_qs; - for (uint ib = sgi; ib < (uint)ne00/64; ib += Q2_0_MV_NSG) { - const float2 d = convert_float2(vload2(j + ib*mh, src0_d)); - float2 acc = 0.0f; + q2_0_mv_acc_t sum = 0.0f; + if (j < mr) { + for (uint ib = sgi; ib < (uint)ne00/64; ib += nsg) { + const q2_0_mv_acc_t d = Q2_0_MV_FLOAT(Q2_0_MV_LOAD_D(j + ib*mr, src0_d)); + q2_0_mv_acc_t acc = 0.0f; for (uint u = 0; u < 16; ++u) { const uint group = ib*16 + u; - const uint q = qs[j + group*mh]; +#if Q2_0_MV_WIDE + const uchar8 q = as_uchar8(((global const uint2 *)src0_qs)[j + group*mr]); +#else + const uchar4 q = as_uchar4(((global const uint *)src0_qs)[j + group*mr]); +#endif const float4 yv = vload4(group, y); - const int4 v0 = convert_int4(((uint4)(q) >> (uint4)(0, 2, 4, 6)) & (uint4)(3)) - (int4)(1); - const int4 v1 = convert_int4(((uint4)(q) >> (uint4)(8, 10, 12, 14)) & (uint4)(3)) - (int4)(1); - acc.s0 += dot(yv, convert_float4(v0)); - acc.s1 += dot(yv, convert_float4(v1)); + acc += (Q2_0_MV_FLOAT( q & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s0; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(2)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s1; + acc += (Q2_0_MV_FLOAT((q >> (q2_0_mv_quant_t)(4)) & (q2_0_mv_quant_t)(3)) - 1.0f)*yv.s2; + acc += (Q2_0_MV_FLOAT( q >> (q2_0_mv_quant_t)(6)) - 1.0f)*yv.s3; } sum += d*acc; } } -#if Q2_0_MV_NSG > 1 - local float2 part[Q2_0_MV_NSG][64]; - part[sgi][lid] = sum; + local q2_0_mv_acc_t part[512]; + part[sgi*lanes + lid] = sum; barrier(CLK_LOCAL_MEM_FENCE); if (sgi != 0) { return; } - for (uint s = 1; s < Q2_0_MV_NSG; ++s) { - sum += part[s][lid]; + for (uint s = 1; s < nsg; ++s) { + sum += part[s*lanes + lid]; } -#endif - if (j < mh) { - vstore2(sum, 0, dst + (ulong)col*(uint)ne0 + row); + if (j < mr) { + Q2_0_MV_STORE(sum, 0, dst + (ulong)col*(uint)ne0 + j*Q2_0_MV_ROWS); } } diff --git a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl index ec71b8756523..1ac250a84b9b 100644 --- a/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl +++ b/ggml/src/ggml-opencl/kernels/mul_mv_f32_f32.cl @@ -116,3 +116,70 @@ kernel void kernel_mul_mat_f32_f32( } } } + +// Each group computes two rows and four columns, with 32 lanes per row. +kernel void kernel_mul_mv_f32_f32_r2( + global const char * src0, + ulong offset0, + global const char * src1, + ulong offset1, + global float * dst, + ulong offsetd, + int ne00, + int ne01, + int ne02, + ulong nb00, + ulong nb01, + ulong nb02, + ulong nb03, + int ne10, + int ne11, + int ne12, + ulong nb10, + ulong nb11, + ulong nb12, + ulong nb13, + int ne0, + int ne1, + int r2, + int r3 +) { + src0 += offset0; + src1 += offset1; + dst = (global float *)((global char *)dst + offsetd); + + const int lid = get_local_id(0); + const int lane = lid % 32; + const int row = get_group_id(0)*2 + lid/32; + const int col = get_group_id(1)*4; + const int im = get_group_id(2); + const int i12 = im % ne12; + const int i13 = im / ne12; + global const float4 * x = (global const float4 *)(src0 + min(row, ne01 - 1)*nb01 + (i12/r2)*nb02 + (i13/r3)*nb03); + global const char * y = src1 + col*nb11 + i12*nb12 + i13*nb13; + + float4 sum = 0.0f; + for (int i = lane; i < ne00/4; i += 32) { + const float4 xv = x[i]; + if (col + 0 < ne11) sum.s0 += dot(xv, ((global const float4 *)(y + 0*nb11))[i]); + if (col + 1 < ne11) sum.s1 += dot(xv, ((global const float4 *)(y + 1*nb11))[i]); + if (col + 2 < ne11) sum.s2 += dot(xv, ((global const float4 *)(y + 2*nb11))[i]); + if (col + 3 < ne11) sum.s3 += dot(xv, ((global const float4 *)(y + 3*nb11))[i]); + } + + local float4 part[64]; + part[lid] = sum; + barrier(CLK_LOCAL_MEM_FENCE); + for (int s = 16; s > 0; s /= 2) { + if (lane < s) { + part[lid] += part[lid + s]; + } + barrier(CLK_LOCAL_MEM_FENCE); + } + if (lane == 0 && row < ne01) { + if (col + 0 < ne11) dst[im*ne1*ne0 + (col + 0)*ne0 + row] = part[lid].s0; + if (col + 1 < ne11) dst[im*ne1*ne0 + (col + 1)*ne0 + row] = part[lid].s1; + if (col + 2 < ne11) dst[im*ne1*ne0 + (col + 2)*ne0 + row] = part[lid].s2; + if (col + 3 < ne11) dst[im*ne1*ne0 + (col + 3)*ne0 + row] = part[lid].s3; + } +} From 162e2bb2e42a0bbf6f9c2d3d8a9eb3cc40b4a4ea Mon Sep 17 00:00:00 2001 From: EugeoSynthesisThirtyTwo Date: Wed, 4 Nov 2026 09:10:22 +0100 Subject: [PATCH 042/286] ggml : fix bf16 (#31609) Co-authored-by: Your Name --- ggml/src/ggml.c | 10 ++++------ tools/export-lora/export-lora.cpp | 5 +++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 8183525aff92..e81148106ec5 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -485,27 +485,25 @@ void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) { } void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_FP32_TO_FP16(x[i]); } } void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) { - int i = 0; - for (; i < n; ++i) { + for (int64_t i = 0; i < n; i++) { y[i] = GGML_BF16_TO_FP32(x[i]); } } void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) { - for (int i = 0; i < n; i++) { + for (int64_t i = 0; i < n; i++) { y[i] = ggml_compute_fp32_to_bf16(x[i]); } } void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) { - int i = 0; + int64_t i = 0; #if defined(__AVX512BF16__) // subnormals are flushed to zero on this platform for (; i + 32 <= n; i += 32) { diff --git a/tools/export-lora/export-lora.cpp b/tools/export-lora/export-lora.cpp index cd7ac39d257f..5ffd97814903 100644 --- a/tools/export-lora/export-lora.cpp +++ b/tools/export-lora/export-lora.cpp @@ -182,6 +182,8 @@ struct lora_merge_ctx { ggml_type get_out_tensor_type(struct ggml_tensor * t) { if (t->type == GGML_TYPE_F32) { return GGML_TYPE_F32; + } else if (t->type == GGML_TYPE_BF16) { + return GGML_TYPE_BF16; } else { return GGML_TYPE_F16; } @@ -409,8 +411,7 @@ struct lora_merge_ctx { static void print_usage(int, char ** argv) { printf("\nexample usage:\n"); - printf("\n %s -m base-model.gguf --lora lora-file.gguf -o merged-model-f16.gguf\n", argv[0]); - printf("\nNOTE: output model is F16\n"); + printf("\n %s -m base-model.gguf --lora lora-file.gguf -o merged-model.gguf\n", argv[0]); printf("\n"); } From 4c4ca5b18854c973655a62a6164a3ae930ee76b0 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 10:32:45 +0200 Subject: [PATCH 043/286] common : do not allow `_` in sidecar (#31639) --- common/download.cpp | 2 +- tests/test-model-resolution.cpp | 15 --------------- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/common/download.cpp b/common/download.cpp index 4de93f30defa..979dd63263c5 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -749,7 +749,7 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, for (const auto & t : tags) { // match the tag as a full token, so ":F16" does not match a BF16 file - std::regex pattern("(^|[-./_])" + t + "[.-]", std::regex::icase); + std::regex pattern("(^|[-./])" + t + "[.-]", std::regex::icase); for (const auto & f : files) { if (gguf_filename_is_model(f.path) && std::regex_search(f.path, pattern)) { diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 73a87482d2c1..e2576a1c9514 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -161,12 +161,6 @@ static const std::vector twins = { "model-Q4_XS.gguf", }; -// an underscore separates the quant like a dash, as in step3p5_flash_Q4_K_S.gguf -static const std::vector underscore = { - "model_Q4_K_S.gguf", - "model_Q8_0.gguf", -}; - // every speculative sidecar type at the same quant static const std::vector quad = { "model-Q8_0.gguf", @@ -301,15 +295,6 @@ static const plan_case plan_cases[] = { "model-IQ4_XS.gguf", {"model-IQ4_XS.gguf"}, "", "", "", "", ""}, - // the token boundary accepts an underscore, for the tag and the default lookup - {"underscore tag", underscore, "test/repo:Q4_K_S", "", false, false, - "model_Q4_K_S.gguf", {"model_Q4_K_S.gguf"}, - "", "", "", "", ""}, - - {"underscore default", underscore, "test/repo", "", false, false, - "model_Q8_0.gguf", {"model_Q8_0.gguf"}, - "", "", "", "", ""}, - // every sidecar type resolves at the tag {"quad exact tag", quad, "test/repo:Q8_0", "", true, false, "model-Q8_0.gguf", {"model-Q8_0.gguf"}, From feaf800060a6633e0ae99b835a260fd5d7d4865f Mon Sep 17 00:00:00 2001 From: "Kirill A. Korinsky" Date: Wed, 4 Nov 2026 10:11:52 +0100 Subject: [PATCH 044/286] ggml : Gate RVV code on the target extension (#31607) The RISC-V vector intrinsic specification requires __riscv_v_intrinsic to be defined when the compiler supports the API, even when the vector extension is not selected. Clang therefore defines it for rv64gc, causing ggml to enter RVV branches where vector types are rejected because the required ISA extensions are absent. Use __riscv_v for optional RVV implementations; this macro represents the selected target ISA. Retain __riscv_v_intrinsic checks in SpaceMIT sources, where they validate mandatory compiler API support rather than select optional vector code. Closes: https://github.com/ggml-org/ggml/issues/1535 --- ggml/src/ggml-cpu/arch/riscv/repack.cpp | 2 +- ggml/src/ggml-cpu/ggml-cpu-impl.h | 2 +- ggml/src/ggml-cpu/ggml-cpu.c | 10 +++++----- ggml/src/ggml-cpu/llamafile/sgemm.cpp | 16 ++++++++-------- ggml/src/ggml-cpu/ops.cpp | 4 ++-- ggml/src/ggml-cpu/simd-gemm.h | 4 ++-- ggml/src/ggml-cpu/simd-mappings.h | 6 ++++-- ggml/src/ggml-cpu/vec.cpp | 14 +++++++------- ggml/src/ggml-cpu/vec.h | 18 +++++++++--------- 9 files changed, 39 insertions(+), 37 deletions(-) diff --git a/ggml/src/ggml-cpu/arch/riscv/repack.cpp b/ggml/src/ggml-cpu/arch/riscv/repack.cpp index c37488cae545..49797df85485 100644 --- a/ggml/src/ggml-cpu/arch/riscv/repack.cpp +++ b/ggml/src/ggml-cpu/arch/riscv/repack.cpp @@ -29,7 +29,7 @@ void ggml_quantize_mat_q8_0_4x8(const float * GGML_RESTRICT x, void * GGML_RESTR assert(k % QK8_0 == 0); const int nb = k / QK8_0; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) block_q8_0x4 * GGML_RESTRICT y = (block_q8_0x4 *) vy; const size_t vl_calc = __riscv_vsetvl_e32m8(QK8_0); const size_t vl_save = __riscv_vsetvl_e64m2(4); diff --git a/ggml/src/ggml-cpu/ggml-cpu-impl.h b/ggml/src/ggml-cpu/ggml-cpu-impl.h index 865847a7785f..f8b363728b24 100644 --- a/ggml/src/ggml-cpu/ggml-cpu-impl.h +++ b/ggml/src/ggml-cpu/ggml-cpu-impl.h @@ -345,7 +345,7 @@ static inline int32x4_t ggml_nvfp4_dot8(const int8x8_t q4_lo, const int8x8_t q8_ #include #endif -#ifdef __riscv_v_intrinsic +#if defined(__riscv_v) #include #endif diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f4eaea5dfe95..c2826fccc49e 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -746,7 +746,7 @@ static void ggml_init_arm_arch_features(void) {} #endif #endif // __ARM_ARCH -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) #include static void ggml_init_riscv_arch_features(void) { ggml_riscv_arch_features.rvv_vlen = __riscv_vlenb(); @@ -3556,7 +3556,7 @@ void ggml_cpu_fp16_to_fp32(const ggml_fp16_t * x, float * y, int64_t n) { _mm_storeu_ps(y + i, y_vec); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfhmin) +#elif defined(__riscv_v) && defined(__riscv_zvfhmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3626,7 +3626,7 @@ void ggml_cpu_bf16_to_fp32(const ggml_bf16_t * x, float * y, int64_t n) { (const __m128i *)(x + i))), 16))); } -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfmin) +#elif defined(__riscv_v) && defined(__riscv_zvfbfmin) // calculate step size const int epr = __riscv_vsetvlmax_e16m2(); const int step = epr * 2; @@ -3746,7 +3746,7 @@ int ggml_cpu_has_arm_fma(void) { } int ggml_cpu_has_riscv_v(void) { -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) return 1; #else return 0; @@ -3754,7 +3754,7 @@ int ggml_cpu_has_riscv_v(void) { } int ggml_cpu_get_rvv_vlen(void) { -#if defined(__riscv) && defined(__riscv_v_intrinsic) +#if defined(__riscv) && defined(__riscv_v) return ggml_riscv_arch_features.rvv_vlen; #else return 0; diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index d28840a91504..b406f375af9d 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -69,7 +69,7 @@ #define VECTOR_REGISTERS 16 #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) #define LMUL 4 #endif @@ -180,7 +180,7 @@ inline float32x4_t madd(float32x4_t a, float32x4_t b, float32x4_t c) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t madd(vfloat32m1_t a, vfloat32m1_t b, vfloat32m1_t c) { return __riscv_vfmacc_vv_f32m1(c, a, b, __riscv_vsetvlmax_e32m1()); } @@ -277,7 +277,7 @@ inline float hsum(__m512 x) { } #endif // __AVX512F__ -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) inline float hsum(vfloat32m1_t x) { return __riscv_vfmv_f_s_f32m1_f32( __riscv_vfredusum_vs_f32m1_f32m1(x, __riscv_vfmv_v_f_f32m1(0, 1), __riscv_vsetvlmax_e32m1())); @@ -458,7 +458,7 @@ template <> inline __m512bh load_partial(const ggml_bf16_t *p, int n) { #endif #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template <> inline vfloat32m1_t load(const float *p) { return __riscv_vle32_v_f32m1(p, __riscv_vsetvlmax_e32m1()); } @@ -503,7 +503,7 @@ template <> inline vbfloat16m4_t load(const ggml_bf16_t *p) { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template T set_zero(); template <> inline vfloat32m1_t set_zero() { @@ -520,7 +520,7 @@ template <> inline vfloat32m8_t set_zero() { } #endif -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template size_t vlmax() { if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m1(); } else if constexpr (std::is_same_v) { return __riscv_vsetvlmax_e32m2(); } @@ -732,7 +732,7 @@ class tinyBLAS { const int64_t ldc; }; -#if defined(__riscv_v_intrinsic) +#if defined(__riscv_v) template class tinyBLAS_RVV { public: @@ -3958,7 +3958,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 params->ith, params->nth}; tb.matmul(m, n); return true; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if LMUL == 1 tinyBLAS_RVV tb{ params, k, (const float *)A, lda, diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index dbdde6997f48..0071283387e0 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -10114,7 +10114,7 @@ static void ggml_compute_forward_ssm_scan_f32( } sumf = GGML_F32xt_REDUCE_ONE(sum); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) // todo: RVV implementation const int np = 0; #else @@ -11778,7 +11778,7 @@ static void ggml_compute_forward_rwkv_wkv7_f32( int64_t h_stride_2d = head_size * head_size; #if defined(GGML_SIMD) - #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v_intrinsic) + #if defined(__ARM_FEATURE_SVE) || defined(__riscv_v) // scalar Route to scalar implementation //TODO: Write SVE code and RVV code for (int64_t t = 0; t < T; t++) { int64_t t_offset = t * t_stride; diff --git a/ggml/src/ggml-cpu/simd-gemm.h b/ggml/src/ggml-cpu/simd-gemm.h index 4b9396d54066..6832444fec1e 100644 --- a/ggml/src/ggml-cpu/simd-gemm.h +++ b/ggml/src/ggml-cpu/simd-gemm.h @@ -5,7 +5,7 @@ #include "simd-mappings.h" // TODO: add support for sizeless vector types -#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v_intrinsic) +#if defined(GGML_SIMD) && !defined(__ARM_FEATURE_SVE) && !defined(__riscv_v) // TODO: untested on avx512 // These are in units of GGML_F32_EPR @@ -149,7 +149,7 @@ static void simd_gemm( C += N; } } -#elif defined(GGML_SIMD) && defined(__riscv_v_intrinsic) +#elif defined(GGML_SIMD) && defined(__riscv_v) // RM accumulators + 1 B vector = RM + 1 <= 8 => RM <= 7 // Microkernel: C[RM x vl] += A[RM x K] * B[K x N] template diff --git a/ggml/src/ggml-cpu/simd-mappings.h b/ggml/src/ggml-cpu/simd-mappings.h index 7b86e8ad9a79..d3d00bf6f717 100644 --- a/ggml/src/ggml-cpu/simd-mappings.h +++ b/ggml/src/ggml-cpu/simd-mappings.h @@ -14,7 +14,9 @@ #include #endif -#if defined(__riscv_v_intrinsic) +// __riscv_v_intrinsic reports compiler API support and can be defined without +// enabling V; use the ISA feature macro to select RVV code. +#if defined(__riscv_v) #include #endif @@ -1306,7 +1308,7 @@ static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { #define GGML_BF16_FMA_HI(acc, x, y) \ (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) // compatible with vlen >= 128 diff --git a/ggml/src/ggml-cpu/vec.cpp b/ggml/src/ggml-cpu/vec.cpp index 8f0bf4999cfb..ed604299a4ce 100644 --- a/ggml/src/ggml-cpu/vec.cpp +++ b/ggml/src/ggml-cpu/vec.cpp @@ -84,7 +84,7 @@ void ggml_vec_dot_f32(int n, float * GGML_RESTRICT s, size_t bs, const float * G } // reduce sum1,sum2 to sum1 GGML_F32_VEC_REDUCE(sumf, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) int vl = __riscv_vsetvlmax_e32m8(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); vfloat32m8_t vsum; @@ -195,7 +195,7 @@ void ggml_vec_dot_bf16(int n, float * GGML_RESTRICT s, size_t bs, ggml_bf16_t * sumf += (ggml_float)_mm_cvtss_f32(g); #undef LOAD -#elif defined(__riscv_v_intrinsic) && defined(__riscv_zvfbfwma) +#elif defined(__riscv_v) && defined(__riscv_zvfbfwma) size_t vl = __riscv_vsetvlmax_e32m4(); // initialize accumulators to all zeroes @@ -318,7 +318,7 @@ void ggml_vec_dot_f16(int n, float * GGML_RESTRICT s, size_t bs, ggml_fp16_t * G sum1_hi = svadd_f32_m(DEFAULT_PG32, sum1_hi, sum3_hi); sumf = ggml_sve_sum_f32x2(sum1_lo, sum1_hi); - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) int vl = __riscv_vsetvlmax_e32m2(); vfloat32m1_t vs = __riscv_vfmv_v_f_f32m1(0.0f, 1); @@ -405,7 +405,7 @@ void ggml_vec_silu_f32(const int n, float * y, const float * x) { for (; i + 3 < n; i += 4) { vec_xst(ggml_v_silu(vec_xl(0, x + i)), 0, y + i); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -446,7 +446,7 @@ void ggml_vec_swiglu_f32(const int n, float * y, const float * x, const float * for (; i + 3 < n; i += 4) { vec_xst(vec_mul(ggml_v_silu(vec_xl(0, x + i)), vec_xl(0, g + i)), 0, y + i); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); vfloat32m2_t vx = __riscv_vle32_v_f32m2(&x[i], vl); @@ -602,7 +602,7 @@ ggml_float ggml_vec_cvar_f32(const int n, float * y, const float * x, const floa val = vec_mul(val, val); sum += (ggml_float)vec_hsum_f32x4(val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int vl; i < n; i += vl) { vl = __riscv_vsetvl_e32m2(n - i); @@ -681,7 +681,7 @@ ggml_float ggml_vec_soft_max_f32(const int n, float * y, const float * x, float vec_xst(v_val, 0, y + i); sum += (ggml_float)vec_hsum_f32x4(v_val); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) vfloat64m1_t vsum = __riscv_vfmv_v_f_f64m1(0, 1); for (int avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m2(n - i); diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index f24fa00f2b0d..aa9b5c80ed40 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -206,7 +206,7 @@ inline static void ggml_vec_dot_f16_unroll(const int n, const int xs, float * GG sumf[0] = ggml_sve_sum_f32x2(sum_0_lo, sum_0_hi); sumf[1] = ggml_sve_sum_f32x2(sum_1_lo, sum_1_hi); np = n; - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) #if defined(__riscv_zvfh) size_t vl = __riscv_vsetvlmax_e32m4(); @@ -396,7 +396,7 @@ inline static void ggml_vec_mad_f32(const int n, float * GGML_RESTRICT y, const svst1_f32(pg, y + np2, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -513,7 +513,7 @@ inline static void ggml_vec_mad_f16(const int n, ggml_fp16_t * GGML_RESTRICT y, svst1_f16(pg, (__fp16 *)(y + np2), hy); } np = n; -#elif defined(__riscv_v_intrinsic) // implies __riscv_v_intrinsic +#elif defined(__riscv_v) #if defined (__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -599,7 +599,7 @@ inline static void ggml_vec_mad_f32_unroll(const int n, const int xs, const int y[i] += x[k][i]*v[k][0]; } } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -660,7 +660,7 @@ inline static void ggml_vec_mad1_f32(const int n, float * y, const float * x, co for (int i = 0; i < n; ++i) { y[i] = x[i]*s + b; } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ax = __riscv_vle32_v_f32m8(&x[i], avl); @@ -729,7 +729,7 @@ inline static void ggml_vec_scale_f32(const int n, float * y, const float v) { ay1 = svmul_f32_m(pg, ay1, vx); svst1_f32(pg, y + i, ay1); } - #elif defined(__riscv_v_intrinsic) + #elif defined(__riscv_v) for (int i = 0, avl; i < n; i += avl) { avl = __riscv_vsetvl_e32m8(n - i); vfloat32m8_t ay = __riscv_vle32_v_f32m8(&y[i], avl); @@ -793,7 +793,7 @@ inline static void ggml_vec_scale_f16(const int n, ggml_fp16_t * y, const float svst1_f16(pg, (__fp16 *)(y + np), out); } np = n; -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) #if defined(__riscv_zvfh) const ggml_fp16_t s = GGML_CPU_FP32_TO_FP16(v); const _Float16 scale = *(const _Float16*)(&s); @@ -1411,7 +1411,7 @@ inline static float32x4_t ggml_v_gelu_quick(float32x4_t x) { return vec_div(x, vec_add(one, exp_cx)); } -#elif defined(__riscv_v_intrinsic) +#elif defined(__riscv_v) // adapted from arm limited optimized routine // the maximum error is 1.45358 plus 0.5 ulps @@ -1469,7 +1469,7 @@ inline static vfloat32m2_t ggml_v_gelu_quick_m2(vfloat32m2_t x, int vl) { return __riscv_vfdiv_vv_f32m2(x, __riscv_vfadd_vf_f32m2(exp_cx, 1.0f, vl), vl); } -#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v_intrinsic +#endif // __ARM_NEON / __AVX2__ / __SSE2__ / __VXE__ / __riscv_v inline static void ggml_vec_silu_f16(const int n, ggml_fp16_t * y, const ggml_fp16_t * x) { for (int i = 0; i < n; ++i) { From e853ec9a8c72e91e8fb0eaa68e1dfe02f3c396f0 Mon Sep 17 00:00:00 2001 From: Aman Gupta Date: Wed, 4 Nov 2026 17:23:25 +0800 Subject: [PATCH 045/286] llama : follow ctx_other devices and flash attention in Gemma4 assistant (#31635) * llama : follow ctx_other devices and flash attention in Gemma4 assistant * llama : include for std::any_of --- src/llama-context.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index eb97569962a0..85c32c995a33 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -14,6 +14,7 @@ #include "llama-sampler.h" #include "llama.h" +#include #include #include #include @@ -332,6 +333,18 @@ llama_context::llama_context( cparams.flash_attn = params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED; cparams.auto_fa = params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_AUTO; + // the KV cache is shared with ctx_other, so follow its flash attention setting instead of resolving it here: + // the probe disables it when the layers and the shared KV are on different devices, which breaks a quantized V cache + if (model.arch == LLM_ARCH_GEMMA4_ASSISTANT) { + const bool flash_attn_other = cparams.ctx_other->get_cparams().flash_attn; + if (!cparams.auto_fa && cparams.flash_attn != flash_attn_other) { + LLAMA_LOG_WARN("%s: flash_attn = %d does not match the context the KV cache is shared with, using %d\n", + __func__, cparams.flash_attn, flash_attn_other); + } + cparams.flash_attn = flash_attn_other; + cparams.auto_fa = false; + } + cparams.fused_gdn_ar = true; cparams.fused_gdn_ch = true; cparams.auto_fgdn = false; @@ -459,6 +472,22 @@ llama_context::llama_context( backends.emplace_back(backend); } + // tensors of ctx_other (e.g. a shared KV cache) can be on devices that this model does not use + if (cparams.ctx_other) { + for (const auto & dev_other : llama_get_model(cparams.ctx_other)->devices) { + const bool found = std::any_of(model.devices.begin(), model.devices.end(), + [&](const llama_device & dev) { return dev.dev == dev_other.dev; }); + if (found) { + continue; + } + ggml_backend_t backend = ggml_backend_dev_init(dev_other.dev, nullptr); + if (backend == nullptr) { + throw std::runtime_error(format("failed to initialize %s backend", ggml_backend_dev_name(dev_other.dev))); + } + backends.emplace_back(backend); + } + } + // add ACCEL backends (such as BLAS) for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { ggml_backend_dev_t dev = ggml_backend_dev_get(i); From cca14f64feebdc34a8909ce6e4882a52f3bc4085 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 11:27:44 +0200 Subject: [PATCH 046/286] ggml : bump version to 0.29.0 (ggml/1680) * ggml : bump version to 0.29.0 * authors : update --- ggml/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index 119669204c98..03fc428a8149 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -4,7 +4,7 @@ project("ggml" C CXX ASM) ### GGML Version set(GGML_VERSION_MAJOR 0) -set(GGML_VERSION_MINOR 28) +set(GGML_VERSION_MINOR 29) set(GGML_VERSION_PATCH 0) set(GGML_VERSION_BASE "${GGML_VERSION_MAJOR}.${GGML_VERSION_MINOR}.${GGML_VERSION_PATCH}") From 13ec8f614cfef964fe97bbf872bc436dbc484a1f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 11:58:40 +0200 Subject: [PATCH 047/286] sync : ggml --- scripts/sync-ggml.last | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/sync-ggml.last b/scripts/sync-ggml.last index 7ecc9a407565..43167cdceab2 100644 --- a/scripts/sync-ggml.last +++ b/scripts/sync-ggml.last @@ -1 +1 @@ -facdc44a67da17df4433adcc468b424f48e9ded2 +57e2c2ca2a7cae756b1c4dfafa0cdfef9051d234 From cd959bae99ff4bed22bef2604f45241c09bbb8c8 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Wed, 4 Nov 2026 14:06:26 +0200 Subject: [PATCH 048/286] llama.cpp : bump version to 0.9.0 (#31646) * llama.cpp : bump version to 0.9.0 * authors : update --- AUTHORS | 27 ++++++++++++++++++++++++++- CMakeLists.txt | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 14921cde5499..96ca4c1d9bff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,4 +1,4 @@ -# date: Wed Oct 28 12:38:26 EET 2026 +# date: Wed Nov 4 12:03:31 EET 2026 # this file is auto-generated by scripts/gen-authors.sh Нияз Гарифзянов <112617865+garrnizon@users.noreply.github.com> @@ -48,6 +48,7 @@ Abheek Gulati abhijain1204fujitsu <139222713+abhijain1204fujitsu@users.noreply.github.com> Abhijit Ramesh abhijitb11 <113058133+abhijitb11@users.noreply.github.com> +Abhijith Jajam <129675815+jajamabhijith@users.noreply.github.com> Abhilash Majumder <30946547+abhilash1910@users.noreply.github.com> Abhinay Krishna Abhiram <78226909+geckguy@users.noreply.github.com> @@ -203,14 +204,17 @@ Ankit Khandelwal Ankur Verma <31362771+ankurvdev@users.noreply.github.com> anon998 <131767832+anon998@users.noreply.github.com> Anri Lombard +Anthony Maltsev <77708730+anthonymaltsev@users.noreply.github.com> Anthony Umfer Anthony Van de Gejuchte Anthony Wang antichristHater <142441588+antichristHater@users.noreply.github.com> Antoine Viallon +Anton I Anton Mitkov Anton Mitkov Antonis Makropoulos +anubhavfujitsu Anudit Nagar Anuj Attri anujj @@ -293,6 +297,7 @@ Bernat Vadell Bernhard M. Wiedemann Bert Wagner Bertay Eren <39909689+bertaye@users.noreply.github.com> +Bhargav Krish Bhavik Sharda <10757940+BLSharda@users.noreply.github.com> bhubbb <79117352+bhubbb@users.noreply.github.com> Bill Sideris @@ -533,6 +538,7 @@ Don Mahurin Dong Won Kim <63934649+ddwkim@users.noreply.github.com> Donghyeon Jeong <54725479+djeong20@users.noreply.github.com> Dongliang Wei <121270393+wdl339@users.noreply.github.com> +Dongsheng Luo Doomsdayrs <38189170+Doomsdayrs@users.noreply.github.com> DooWoong Lee (David) DorianRudolph @@ -612,6 +618,7 @@ Faez Shakil fairydreaming <166155368+fairydreaming@users.noreply.github.com> Faisal Zaghloul Faisal Zaghloul +FamousMonsterr Fan Shupei FantasyGmm <16450052+FantasyGmm@users.noreply.github.com> fanyang @@ -654,6 +661,7 @@ Frederik Vogel Fredrik Hultin fredzillman frob +frogscooper Frost-54 <91168148+Frost-54@users.noreply.github.com> Frosty40 Funtowicz Morgan @@ -694,6 +702,7 @@ Gilad S. <7817232+giladgd@users.noreply.github.com> Giovanni Rivera github-actions[bot] GittyBurstein +Giulianno Giuseppe Scrivano Giuseppe Scrivano GiviMAD @@ -762,6 +771,7 @@ Henry147147 <44851451+Henry147147@users.noreply.github.com> Herman Semenoff Herman Semenov Hesen Peng +hesiru hey-gm HighDoping HimariO @@ -919,6 +929,7 @@ Jesse Ikonen Jesse Jojo Johnson Jesse LaRose Jesse Posner +Jesse Symons <155509533+Torchit1@users.noreply.github.com> Jesus Gulfo Jesus Talavera <145992175+jesus-talavera-ibm@users.noreply.github.com> Jetson Tan @@ -1071,6 +1082,7 @@ Kevin Liu <4396kevinliu@gmail.com> Kevin Lo Kevin Pouget Kevin Wang +Keyu Zhu Khashayar Ghafouri <43180261+khashayarghafouri@users.noreply.github.com> khimaros Kilian Hu <90606809+kilian-hu@users.noreply.github.com> @@ -1078,6 +1090,7 @@ Kilian Krampf kiltyj Kim S. kimminsu <80271594+kimminsu38oo@users.noreply.github.com> +Kirill A. Korinsky KITAITI Makoto kiwi <122582483+kiwi142857@users.noreply.github.com> klosax <131523366+klosax@users.noreply.github.com> @@ -1136,6 +1149,7 @@ leejet Leng Yue Lennart Austenfeld <53152202+l-austenfeld@users.noreply.github.com> leo-pony +Leon <118652405+LeON-Nie-code@users.noreply.github.com> Leon Knauer Leonard Mosescu leonardHONG <2695316095@qq.com> @@ -1146,8 +1160,10 @@ lexasub lgai-exaone lhez lhez +Li Chuyang <1418587903@qq.com> Li Pengzhan <151381994+Lpzhan931@users.noreply.github.com> Li Tan +li-lizhe <82501526+li-lizhe@users.noreply.github.com> liangliangchang liminfei-amd <91481003+liminfei-amd@users.noreply.github.com> limitedAtonement @@ -1166,6 +1182,7 @@ LoganDark Loïc Carrère lon <114724657+longregen@users.noreply.github.com> loonerin <132926317+loonerin@users.noreply.github.com> +Lorenzo Bona losewayy <1134285612@qq.com> LostRuins <39025047+LostRuins@users.noreply.github.com> LostRuins Concedo <39025047+LostRuins@users.noreply.github.com> @@ -1253,6 +1270,7 @@ Masaya, Kato <62578291+msy-kato@users.noreply.github.com> mashdragon <122402293+mashdragon@users.noreply.github.com> Mason Milburn MasterYi1024 <39848311+MasterYi1024@users.noreply.github.com> +Maternion <98753158+maternion@users.noreply.github.com> Mateusz Charytoniuk Matheus C. França Matheus Gabriel Alves Silva @@ -1350,6 +1368,7 @@ Miwa / Ensan <63481257+ensan-hcl@users.noreply.github.com> miyan <1138989048@qq.com> mj-shifu <77107165+mj-shifu@users.noreply.github.com> mkoker <132301062+mkoker@users.noreply.github.com> +mmalpe-nv mmyjona mnehete32 <33429707+mnehete32@users.noreply.github.com> Mohamed Elashri @@ -1368,6 +1387,7 @@ mtmcp <141645996+mtmcp@users.noreply.github.com> muggle-stack Muhammad Saad Muhammad Salem +MuoDoo <36472129+MuoDoo@users.noreply.github.com> Murilo Santana Musab Gultekin musoles <135031143+musoles@users.noreply.github.com> @@ -1416,10 +1436,12 @@ Nicolas Budyn <16596345+nbud@users.noreply.github.com> Nicolas Mowen Nicolás Pérez Nicolò Scipione +Nidhi Nigel Bosch Nigel Jones Nik Bogatyrev <50734723+notforu777@users.noreply.github.com> Nikhil Jain +Nikhil Sharma <150424968+nikhilfujitsu@users.noreply.github.com> Nikita Gordeev Nikita Sarychev <42014488+sARY77@users.noreply.github.com> Niklas Korz @@ -1592,6 +1614,7 @@ Raman Shinde Rand Xie Randall Fitzgerald Random Fly +randomized <286843573@qq.com> rankaiyx rankaiyx RapidMark <32768622+RapidMark@users.noreply.github.com> @@ -2009,6 +2032,7 @@ Tungsten842 Tushar tv1wnd <55383215+tv1wnd@users.noreply.github.com> tyronecai +Uber <20715993+YagUber@users.noreply.github.com> ubergarm ubik2 UEXTM.com <84163508+uextm@users.noreply.github.com> @@ -2242,4 +2266,5 @@ Zijun Yu zql <37731799+zqlcode@users.noreply.github.com> zrm Zsapi +zyang-dev <267119621+zyang-dev@users.noreply.github.com> zzzzwc diff --git a/CMakeLists.txt b/CMakeLists.txt index e37c8aa4cbe7..b8b40e75f68f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ include(CheckIncludeFileCXX) ### llama.cpp version set(LLAMA_VERSION_MAJOR 0) -set(LLAMA_VERSION_MINOR 8) +set(LLAMA_VERSION_MINOR 9) set(LLAMA_VERSION_PATCH 0) set(LLAMA_VERSION_BASE "${LLAMA_VERSION_MAJOR}.${LLAMA_VERSION_MINOR}.${LLAMA_VERSION_PATCH}") From 52e53a76df0298d2c316ae7e704eeec03b0bf478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Gallou=C3=ABt?= Date: Wed, 4 Nov 2026 14:21:38 +0100 Subject: [PATCH 049/286] common : delete common_get_path_from_env() (#31444) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Adrien Gallouët --- common/arg.cpp | 2 +- common/common.cpp | 23 ++++++----------------- common/common.h | 3 --- common/hf-cache.cpp | 2 +- tools/server/server-tools.cpp | 9 +++------ 5 files changed, 11 insertions(+), 28 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index e3ab084c67f9..67b492e113b1 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -710,7 +710,7 @@ static void common_params_apply_system_config(common_params & params, llama_exam std::vector paths; #if defined(_WIN32) - const std::filesystem::path program_data = common_get_path_from_env("PROGRAMDATA"); + const std::filesystem::path program_data = common_get_env("PROGRAMDATA"); if (!program_data.empty()) { paths.push_back(program_data / "llama.cpp" / "config.ini"); } diff --git a/common/common.cpp b/common/common.cpp index 95a2dceef65d..20285b6f7246 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -983,20 +983,9 @@ void common_set_env(const std::string & name, const std::string & value) { #endif } -std::filesystem::path common_get_path_from_env(const std::string & name) { -#if defined(_WIN32) - const std::wstring wname = utf8_to_wstring(name); - const wchar_t * wvalue = _wgetenv(wname.c_str()); - return wvalue ? std::filesystem::path(wvalue) : std::filesystem::path(); -#else - const char * value = std::getenv(name.c_str()); - return value ? std::filesystem::path(value) : std::filesystem::path(); -#endif -} - #if !defined(_WIN32) static std::filesystem::path get_home_directory() { - std::filesystem::path home = common_get_path_from_env("HOME"); + std::filesystem::path home = common_get_env("HOME"); if (!home.empty()) { return home; } @@ -1009,19 +998,19 @@ static std::filesystem::path get_home_directory() { #endif std::filesystem::path fs_get_cache_directory() { - std::filesystem::path cache_directory = common_get_path_from_env("LLAMA_CACHE"); + std::filesystem::path cache_directory = common_get_env("LLAMA_CACHE"); if (!cache_directory.empty()) { return cache_directory; } #if defined(_WIN32) - cache_directory = common_get_path_from_env("LOCALAPPDATA"); + cache_directory = common_get_env("LOCALAPPDATA"); if (cache_directory.empty()) { throw std::runtime_error("Failed to find %LOCALAPPDATA% directory"); } #elif defined(__APPLE__) cache_directory = get_home_directory() / "Library/Caches"; #else - cache_directory = common_get_path_from_env("XDG_CACHE_HOME"); + cache_directory = common_get_env("XDG_CACHE_HOME"); if (cache_directory.empty()) { cache_directory = get_home_directory() / ".cache"; } @@ -1032,12 +1021,12 @@ std::filesystem::path fs_get_cache_directory() { std::filesystem::path fs_get_config_directory() { std::filesystem::path config_directory; #if defined(_WIN32) - config_directory = common_get_path_from_env("APPDATA"); + config_directory = common_get_env("APPDATA"); if (config_directory.empty()) { throw std::runtime_error("Failed to find %APPDATA% directory"); } #else - config_directory = common_get_path_from_env("XDG_CONFIG_HOME"); + config_directory = common_get_env("XDG_CONFIG_HOME"); if (config_directory.empty()) { config_directory = get_home_directory() / ".config"; } diff --git a/common/common.h b/common/common.h index e970fb39b25d..6ee90c1fe270 100644 --- a/common/common.h +++ b/common/common.h @@ -919,9 +919,6 @@ std::string wstring_to_utf8(const std::wstring & str); std::string common_get_env(const std::string & name); void common_set_env(const std::string & name, const std::string & value); -// reads a path from the environment, an unset variable gives an empty path -std::filesystem::path common_get_path_from_env(const std::string & name); - // // Filesystem utils // diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index aafc940c5789..a7f328fea3ef 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -45,7 +45,7 @@ static fs::path get_cache_directory() { {HOME_DIR, fs::path(".cache") / "huggingface" / "hub"} }; for (const auto & entry : entries) { - if (fs::path base = common_get_path_from_env(entry.var); !base.empty()) { + if (fs::path base = common_get_env(entry.var); !base.empty()) { return entry.path.empty() ? base : base / entry.path; } } diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index ad6dd1d781ab..c1cf27cd22d2 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -128,13 +128,10 @@ static std::string path_to_utf8(const fs::path & p) { static const std::string & home_dir() { static const std::string home = [] { #ifdef _WIN32 - // the narrow getenv would return the profile path in the active code page - const wchar_t * w = _wgetenv(L"HOME"); - if (w == nullptr) w = _wgetenv(L"USERPROFILE"); - return w ? path_to_utf8(fs::path(w)) : std::string(); + const std::string h = common_get_env("HOME"); + return h.empty() ? common_get_env("USERPROFILE") : h; #else - const char * h = getenv("HOME"); - return h ? std::string(h) : std::string(); + return common_get_env("HOME"); #endif }(); return home; From 034e2535a90718762d3bda169ff1a06bcf3e5d1e Mon Sep 17 00:00:00 2001 From: Aaron Thomas <84493696+arxhr007@users.noreply.github.com> Date: Wed, 4 Nov 2026 19:12:39 +0530 Subject: [PATCH 050/286] gguf-py : validate every tokenizer merge pair, not only the first (#31631) * gguf-py : validate every tokenizer merge pair, not only the first The transformers 4.45 pair format was detected by checking merges[0], then every element was converted unchecked: a non-pair element raised a bare TypeError, and pairs with the wrong arity or non-string parts were silently mangled into invalid merges. Validate each element (2-string list) up front and raise the existing "Unknown tokenizer merges format" error for anything else. Assisted-by: Devin * gguf-py : remove test_vocab.py per review --- gguf-py/gguf/vocab.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index 3b82c3961496..d299cfd0019c 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -221,10 +221,12 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: if isinstance(merges, list) and merges: if isinstance(merges[0], str): self.merges = merges - elif isinstance(merges[0], list) and len(merges[0]) == 2 and isinstance(merges[0][0], str): + elif isinstance(merges[0], list): # New format since transformers 4.45 to support spaces in merges # ref: https://github.com/ggml-org/llama.cpp/issues/9692 # TODO: internally store as the new format instead of converting to old + if not all(isinstance(pair, list) and len(pair) == 2 and all(isinstance(s, str) for s in pair) for pair in merges): + raise ValueError("Unknown tokenizer merges format") if any(' ' in s for pair in merges for s in pair): logger.warning(f'Spaces in merges detected, encoding as {chr(ord(" ") + 256)!r}') self.merges = [ From 00df7c22d9b714b5647c1450328961868a3a4270 Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Wed, 4 Nov 2026 21:45:29 +0800 Subject: [PATCH 051/286] sycl : fix docker issue (#31605) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add oneDNN in build image * Update .devops/intel.Dockerfile Co-authored-by: Sigbjørn Skjæret --------- Co-authored-by: Sigbjørn Skjæret --- .devops/intel.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devops/intel.Dockerfile b/.devops/intel.Dockerfile index 09daa5fe5d1a..2ea77b3661b0 100644 --- a/.devops/intel.Dockerfile +++ b/.devops/intel.Dockerfile @@ -64,7 +64,7 @@ RUN mkdir -p /app/full \ && cp requirements.txt /app/full \ && cp .devops/tools.sh /app/full/tools.sh -FROM docker.io/intel/oneapi-toolkit:$ONEAPI_VERSION AS base +FROM docker.io/intel/deep-learning-essentials:$ONEAPI_VERSION AS base ARG BUILD_DATE=N/A ARG APP_VERSION=N/A @@ -79,6 +79,8 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \ org.opencontainers.image.url=$IMAGE_URL \ org.opencontainers.image.source=$IMAGE_SOURCE +COPY --from=build /opt/intel/oneapi/dnnl /opt/intel/oneapi/dnnl + #Following versions are for multiple GPUs, since 26.x has known issue: # https://github.com/ggml-org/llama.cpp/issues/21747, # https://github.com/intel/compute-runtime/issues/921. From b393d73bb0448ad101fe84cc77f3eee465fb947d Mon Sep 17 00:00:00 2001 From: Masashi Yoshimura Date: Wed, 4 Nov 2026 22:46:06 +0900 Subject: [PATCH 052/286] webgpu: return maxBufferSize from get_max_size and avoid whole-buffer bindings in memset/clear (#31348) * Fix get_max_size and improve the memset shader * Fix buffer_memset to handle the buffer size more than max binding size. --- ggml/src/ggml-webgpu/ggml-webgpu.cpp | 97 ++++++++++--------- ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl | 43 ++++---- 2 files changed, 73 insertions(+), 67 deletions(-) diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cpp index bb2deb9da3f0..ea3d91f3b74b 100644 --- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp +++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp @@ -182,7 +182,6 @@ struct webgpu_capabilities { uint32_t subgroup_size = 0; uint32_t min_subgroup_size = 0; uint32_t max_subgroup_size = 0; - size_t memset_bytes_per_thread; }; // Stores global webgpu members @@ -256,8 +255,6 @@ struct webgpu_context_struct { wgpu::ComputePassEncoder active_compute_pass; bool batch_compute_passes = true; - size_t memset_bytes_per_thread; - #ifdef GGML_WEBGPU_GPU_PROFILE // Profiling: per-shader GPU time in ms std::unordered_map shader_gpu_time_ms; @@ -672,38 +669,54 @@ static void ggml_backend_webgpu_buffer_memset(webgpu_global_context & ctx, size_t size) { std::lock_guard lock(ctx->mutex); - std::vector params = { (uint32_t) offset, (uint32_t) size, value }; - std::vector entries = { ggml_webgpu_make_bind_group_entry(0, buf, 0, buf.GetSize()) }; - size_t bytes_per_wg = - ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * ctx->capabilities.memset_bytes_per_thread; - uint32_t wg_x = CEIL_DIV(size + 3, bytes_per_wg); - - ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); - - wgpu::BindGroupEntry params_entry = {}; - params_entry.binding = 1; - params_entry.buffer = ctx->memset_params_buf; - params_entry.offset = 0; - params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; - entries.push_back(params_entry); - - wgpu::BindGroupDescriptor bind_group_desc; - bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); - bind_group_desc.entryCount = entries.size(); - bind_group_desc.entries = entries.data(); - bind_group_desc.label = ctx->memset_pipeline.name.c_str(); - wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); - - wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); - wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); - pass.SetPipeline(ctx->memset_pipeline.pipeline); - pass.SetBindGroup(0, bind_group); - pass.DispatchWorkgroups(wg_x, 1, 1); - pass.End(); - - wgpu::CommandBuffer command = encoder.Finish(); - std::vector commands = { command }; - ctx->queue.Submit(commands.size(), commands.data()); + const size_t max_bind_size = ctx->capabilities.limits.maxStorageBufferBindingSize; + const size_t max_wg_size = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; + + while (size > 0) { + const size_t align_offset = offset & ~(ctx->capabilities.limits.minStorageBufferOffsetAlignment - 1); + const size_t lead = offset - align_offset; + const size_t chunk_size = std::min(size, max_bind_size - lead); + const size_t bind_size = ROUNDUP_POW2(chunk_size + lead, WEBGPU_STORAGE_BUF_BINDING_MULT); + + std::vector params = { (uint32_t) lead, (uint32_t) chunk_size, value }; + std::vector entries = { ggml_webgpu_make_bind_group_entry(0, buf, align_offset, + bind_size) }; + uint32_t wg_x; + uint32_t wg_y; + uint32_t n_values = CEIL_DIV(chunk_size + lead, sizeof(uint32_t)); + uint32_t total_wg = CEIL_DIV(n_values, max_wg_size); + compute_2d_workgroups(total_wg, ctx->capabilities.limits.maxComputeWorkgroupsPerDimension, wg_x, wg_y); + + ctx->queue.WriteBuffer(ctx->memset_params_buf, 0, params.data(), params.size() * sizeof(uint32_t)); + + wgpu::BindGroupEntry params_entry = {}; + params_entry.binding = 1; + params_entry.buffer = ctx->memset_params_buf; + params_entry.offset = 0; + params_entry.size = WEBGPU_PARAMS_BUF_SIZE_BYTES; + entries.push_back(params_entry); + + wgpu::BindGroupDescriptor bind_group_desc; + bind_group_desc.layout = ctx->memset_pipeline.pipeline.GetBindGroupLayout(0); + bind_group_desc.entryCount = entries.size(); + bind_group_desc.entries = entries.data(); + bind_group_desc.label = ctx->memset_pipeline.name.c_str(); + wgpu::BindGroup bind_group = ctx->device.CreateBindGroup(&bind_group_desc); + + wgpu::CommandEncoder encoder = ctx->device.CreateCommandEncoder(); + wgpu::ComputePassEncoder pass = encoder.BeginComputePass(); + pass.SetPipeline(ctx->memset_pipeline.pipeline); + pass.SetBindGroup(0, bind_group); + pass.DispatchWorkgroups(wg_x, wg_y, 1); + pass.End(); + + wgpu::CommandBuffer command = encoder.Finish(); + std::vector commands = { command }; + ctx->queue.Submit(commands.size(), commands.data()); + + offset += chunk_size; + size -= chunk_size; + } } /** End WebGPU Actions */ @@ -4006,12 +4019,10 @@ static size_t ggml_backend_webgpu_buffer_type_get_alignment(ggml_backend_buffer_ return dev_ctx->webgpu_global_ctx->capabilities.limits.minStorageBufferOffsetAlignment; } -// maxBufferSize might be larger, but you can't bind more than -// maxStorageBufferBindingSize to a single binding. static size_t ggml_backend_webgpu_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) { ggml_backend_webgpu_device_context * dev_ctx = static_cast(buft->device->context); - return dev_ctx->webgpu_global_ctx->capabilities.limits.maxStorageBufferBindingSize; + return dev_ctx->webgpu_global_ctx->capabilities.limits.maxBufferSize; } static size_t ggml_backend_webgpu_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, @@ -4176,17 +4187,9 @@ static ggml_guid_t ggml_backend_webgpu_guid(void) { } static void ggml_webgpu_init_memset_pipeline(webgpu_global_context & ctx) { - // we use the maximum workgroup size for the memset pipeline - size_t max_threads = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup * - ctx->capabilities.limits.maxComputeWorkgroupsPerDimension; - // Size the bytes_per_thread so that the largest buffer size can be handled - ctx->capabilities.memset_bytes_per_thread = - CEIL_DIV(ctx->capabilities.limits.maxStorageBufferBindingSize, max_threads); - std::vector constants(2); + std::vector constants(1); constants[0].key = "wg_size"; constants[0].value = ctx->capabilities.limits.maxComputeInvocationsPerWorkgroup; - constants[1].key = "bytes_per_thread"; - constants[1].value = ctx->capabilities.memset_bytes_per_thread; ctx->memset_pipeline = ggml_webgpu_create_pipeline(ctx->device, wgsl_memset, "memset", constants); } diff --git a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl index 194d2d6f58c7..af5a23bc0f69 100644 --- a/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl +++ b/ggml/src/ggml-webgpu/wgsl-shaders/memset.wgsl @@ -11,30 +11,33 @@ struct Params { var params: Params; override wg_size: u32; -override bytes_per_thread: u32; @compute @workgroup_size(wg_size) -fn main(@builtin(global_invocation_id) gid: vec3) { - let i = gid.x * bytes_per_thread; +fn main(@builtin(workgroup_id) wid: vec3, + @builtin(num_workgroups) num_wg: vec3, + @builtin(local_invocation_id) lid: vec3) { + + let i = (wid.x + wid.y * num_wg.x) * wg_size + lid.x; let start = params.offset; let end = params.offset + params.size; + let id = (start >> 2u) + i; + let id_u32 = id * 4; + + if (id_u32 >= end) { + return; + } + + let lo = max(start, id_u32) - id_u32; + let hi = min(end, id_u32 + 4u) - id_u32; + var mask = 0xFFFFFFFFu << (8u * lo); + if (hi < 4u) { + mask &= (1u << (8u * hi)) - 1u; + } - for (var j: u32 = 0u; j < bytes_per_thread; j += 4) { - let byte_index = start + i + j; - if (byte_index + 4 <= end) { - output_buffer[byte_index >> 2] = params.value; - } else { - // Handle tail (unaligned) - for (var k: u32 = 0; k < 4; k++) { - let idx = byte_index + k; - if (idx < end) { - let word_idx = idx >> 2; - let bit_offset = (idx & 3) * 8u; - let mask = ~(0xffu << bit_offset); - let existing = output_buffer[word_idx]; - output_buffer[word_idx] = (existing & mask) | (params.value & (0xffu << bit_offset)); - } - } - } + if (mask == 0xFFFFFFFFu) { + output_buffer[id] = params.value; + } else { + let existing = output_buffer[id]; + output_buffer[id] = (existing & ~(mask)) | (params.value & mask); } } From efbf6e80a5c121584736b5face7cfced9da707dd Mon Sep 17 00:00:00 2001 From: Aleksander Grygier Date: Wed, 4 Nov 2026 14:46:51 +0100 Subject: [PATCH 053/286] ui: show text and page thumbnails for PDF attachments, stop endless conversion retries (#31468) * fix(ui): show extracted text in PDF attachment preview Assisted-by: Claude (Opus 5.5 Medium) * feat(ui): show page thumbnails for PDF attachments sent as images Assisted-by: Claude (Opus 5.5 Medium) * ui: stop retrying PDF preview conversion after it fails A failed PDF-to-image conversion or text extraction reset its loading flag, which re-ran the effect and retried forever. Remember the failed item for each view and only retry when the item changes, so the error state stays on screen. Add a PdfConversionError story that feeds a broken PDF. Assisted-by: Claude (Opus 5.5 Medium) * ui: keep the text of PDFs sent as images and tidy the PDF preview Store the extracted text of a PDF sent as images in its content, so the Text view reads it instead of parsing the PDF again; only messages stored with the old placeholder extract it. The model still receives the images only. Guard retries on the error state, as the item remounts per id, skip an empty base64 payload, and share the status markup between views. Assisted-by: Claude (Opus 5.5 Medium) --- .../ChatAttachmentsListItem.svelte | 18 +- ...atAttachmentsListItemThumbnailImage.svelte | 10 + .../ChatAttachmentsPreview.svelte | 2 + ...hatAttachmentsPreviewCurrentItemPdf.svelte | 201 +++++++++++------- ...hatAttachmentsPreviewThumbnailStrip.svelte | 3 +- .../supported-file-types.constants.ts | 3 + tools/ui/src/lib/utils/attachment-display.ts | 11 +- .../src/lib/utils/convert-files-to-extra.ts | 4 +- .../ChatAttachmentsPreview.stories.svelte | 47 ++++ 9 files changed, 217 insertions(+), 82 deletions(-) create mode 100644 tools/ui/tests/stories/chat/screen/attachments/ChatAttachmentsPreview.stories.svelte diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 754856d60b86..b62cbc2dbae8 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -106,18 +106,22 @@ {readonly} width={imageWidth} /> -{:else if isPdfFile(item.attachment, item.uploadedFile)} - 0 ? `PDF, ${pageCount} ${pageCount === 1 ? 'page' : 'pages'}` : 'PDF'} name={item.name} onRemove={onFileRemove} onclick={() => onPreview?.(item)} + preview={item.preview} {readonly} - size={item.size} - textContent={item.textContent} - uploadedFile={item.uploadedFile} + width={imageWidth} /> {:else} void; onRemove?: (id: string) => void; name: string; @@ -20,6 +21,7 @@ height = 'h-16', id, imageClass = '', + label, name, onclick, onRemove, @@ -49,6 +51,14 @@ {@render image()} {/if} + {#if label} + + {label} + + {/if} + {#if !readonly}
(PdfViewMode.PAGES); let pdfImages = $state([]); let pdfImagesLoading = $state(false); + // a failure stays on screen: the effect does not retry while an error is set let pdfImagesError = $state(null); + let pdfText = $state(null); + let pdfTextLoading = $state(false); + let pdfTextError = $state(null); let language = $derived(getLanguageFromFilename(displayName)); + // older PDFs sent as images stored a placeholder in `content`, so their text is extracted again + let hasStoredText = $derived( + currentItem?.attachment?.type === AttachmentType.PDF + ? !( + currentItem.attachment.processedAsImages && + PDF_AS_IMAGES_LEGACY_CONTENT_REGEX.test(currentItem.attachment.content) + ) + : !!displayTextContent + ); + + let isLoading = $derived(pdfImagesLoading || pdfTextLoading); + + function getPdfFile(): File | null { + if (currentItem?.uploadedFile?.file) { + return currentItem.uploadedFile.file; + } + + if ( + currentItem?.attachment && + 'base64Data' in currentItem.attachment && + currentItem.attachment.base64Data + ) { + const byteArray = Uint8Array.from(atob(currentItem.attachment.base64Data), (char) => + char.charCodeAt(0) + ); + + return new File([byteArray], displayName, { type: MimeTypeApplication.PDF }); + } + + return null; + } + async function loadPdfImages() { if (pdfImages.length > 0 || pdfImagesLoading || !currentItem) return; + if ( + currentItem.attachment?.type === AttachmentType.PDF && + currentItem.attachment.images?.length + ) { + pdfImages = currentItem.attachment.images; + + return; + } + + if (pdfImagesError) return; + pdfImagesLoading = true; - pdfImagesError = null; try { - let file: File | null = null; - - if (currentItem.uploadedFile?.file) { - file = currentItem.uploadedFile.file; - } else if (currentItem.attachment) { - // Check if we have pre-processed images - if ( - 'images' in currentItem.attachment && - currentItem.attachment.images && - Array.isArray(currentItem.attachment.images) && - currentItem.attachment.images.length > 0 - ) { - pdfImages = currentItem.attachment.images; - - return; - } - - // Convert base64 back to File for processing - if ('base64Data' in currentItem.attachment && currentItem.attachment.base64Data) { - const base64Data = currentItem.attachment.base64Data; - const byteCharacters = atob(base64Data); - const byteNumbers = new Array(byteCharacters.length); - - for (let i = 0; i < byteCharacters.length; i++) { - byteNumbers[i] = byteCharacters.charCodeAt(i); - } - const byteArray = new Uint8Array(byteNumbers); - - file = new File([byteArray], displayName, { type: 'application/pdf' }); - } - } + const file = getPdfFile(); - if (file) { - pdfImages = await convertPDFToImage(file); - } else { + if (!file) { throw new Error('No PDF file available for conversion'); } + + pdfImages = await convertPDFToImage(file); } catch (error) { pdfImagesError = error instanceof Error ? error.message : 'Failed to load PDF images'; } finally { @@ -78,26 +94,62 @@ } } + async function loadPdfText() { + if (pdfText !== null || pdfTextLoading || !currentItem) return; + + if (hasStoredText) { + pdfText = displayTextContent ?? ''; + + return; + } + + if (pdfTextError) return; + + pdfTextLoading = true; + + try { + const file = getPdfFile(); + + if (!file) { + throw new Error('No PDF file available for text extraction'); + } + + pdfText = await convertPDFToText(file); + } catch (error) { + pdfTextError = error instanceof Error ? error.message : 'Failed to extract PDF text'; + } finally { + pdfTextLoading = false; + } + } + $effect(() => { if (pdfViewMode === PdfViewMode.PAGES) { loadPdfImages(); + } else { + loadPdfText(); } });
+ favorited + {/if} + + - {#if sidecarBadges?.length} - -
- {#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} - - {/each} -
- {/if} -
+
+ {#each sidecarBadges as badge (badge.repo + ':' + badge.kind + ':' + (badge.quant ?? ''))} + + {/each} +
+ {/if} +
- + - + - {#if download} - - {/if} + {#if download} + + {/if} -
- +
+ +
- + diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte index 6838a85da73d..e99cbea2caf1 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsManager/ModelsManagerQuantRow.svelte @@ -5,7 +5,7 @@ import { modelRowActions } from './row-actions'; import { configuredContext, isRowControlClick } from './utils'; import { MoreHorizontal } from '@lucide/svelte'; - import { DropdownMenuActions } from '$lib/components/app'; + import { ContextMenuActions, DropdownMenuActions } from '$lib/components/app'; import { Badge } from '$lib/components/ui/badge'; import { MODEL_ROW_GRID_CLASS, MODEL_ROW_TRAILING_CELL_CLASS } from '$lib/constants'; import { modelsStore } from '$lib/stores'; @@ -26,53 +26,62 @@ let favorite = $derived(isFavorite(option)); let isHidden = $derived(modelsStore.isHidden(option.id)); let quant = $derived(option.parsedId?.quantization ?? option.model); + + // the "more actions" dropdown and the right-click menu offer the same actions + let actions = $derived(modelRowActions(option, favorite, isHidden, onDelete)); + let contextMenuOpen = $state(false); - - - -
{ - if (!isRowControlClick(event)) onSelect(option); - }} -> - - + {option.model} + - + - + -
- +
+ +
-
+ diff --git a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts index 42c282e05c5d..b98ab6bdf9b7 100644 --- a/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts +++ b/tools/ui/src/lib/components/app/models/ModelsManager/row-actions.ts @@ -2,6 +2,7 @@ import { Compass, Eye, EyeOff, Heart, HeartOff, Trash2 } from '@lucide/svelte'; import { MODEL_DOWNLOAD_ICONS } from '$lib/constants'; import { ModelRowDownloadState } from '$lib/enums'; import { modelsStore, uiStore } from '$lib/stores'; +import type { MenuAction } from '$lib/types'; import type { ModelOption } from '$lib/types/models'; import { repoOf } from '$lib/utils'; @@ -12,7 +13,7 @@ export function modelRowActions( onDelete: (option: ModelOption) => void, /** Download state, when the row stands for a tracked download. */ download?: ModelRowDownloadState | null -) { +): MenuAction[] { // the details pane is keyed by repo, not by `:`; the entry exists only // while Discover is on const viewInDiscover = uiStore.isDiscoverEnabled diff --git a/tools/ui/src/lib/components/app/navigation/ContextMenuActions.svelte b/tools/ui/src/lib/components/app/navigation/ContextMenuActions.svelte new file mode 100644 index 000000000000..ca0f7de60c99 --- /dev/null +++ b/tools/ui/src/lib/components/app/navigation/ContextMenuActions.svelte @@ -0,0 +1,31 @@ + + + + + + {@render children()} + + + + + + diff --git a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte index f3143050693e..49d38d45dde2 100644 --- a/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte +++ b/tools/ui/src/lib/components/app/navigation/DropdownMenuActions.svelte @@ -1,29 +1,20 @@ + +{#each actions as action, index (action.label)} + {#if action.separator && index > 0} + + {/if} + + {#if action.tooltip} + + + + {#snippet child({ props })} +
+ {@render menuItem(action)} +
+ {/snippet} +
+ + +

{action.tooltip}

+
+
+ {:else} + {@render menuItem(action)} + {/if} +{/each} + +{#snippet menuItem(action: MenuAction)} + +
+ {@render iconComponent( + action.icon, + `h-4 w-4 ${action.variant === 'destructive' ? 'text-destructive' : ''}` + )} + {action.label} +
+ + {#if action.shortcut} + + {/if} +
+{/snippet} + +{#snippet iconComponent(IconComponent: Component, className: string)} + +{/snippet} diff --git a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte index 5db6066b0509..e4fc91cb8d62 100644 --- a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte +++ b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte @@ -12,7 +12,7 @@ Square, Trash2 } from '@lucide/svelte'; - import { DropdownMenuActions } from '$lib/components/app'; + import { ContextMenuActions, DropdownMenuActions } from '$lib/components/app'; import { TruncatedText } from '$lib/components/app'; import { Checkbox } from '$lib/components/ui/checkbox'; import * as Tooltip from '$lib/components/ui/tooltip'; @@ -26,6 +26,7 @@ } from '$lib/constants'; import { RouterService } from '$lib/services/router.service'; import { chatStore, conversationsStore, settingsStore, tabsStore } from '$lib/stores'; + import type { MenuAction } from '$lib/types'; import { onMount } from 'svelte'; interface Props { @@ -62,6 +63,7 @@ let renderActionsDropdown = $state(false); let dropdownOpen = $state(false); + let contextMenuOpen = $state(false); let isLoading = $derived(chatStore.getAllLoadingChats().includes(conversation.id)); @@ -94,6 +96,60 @@ openInNewTab(); } + // the "more actions" dropdown and the right-click menu offer the same actions; the + // shortcuts act on the active conversation, so only its row shows them + let actions = $derived([ + ...(canOpenInNewTab + ? [ + { + icon: PanelTopOpen, + label: 'Open in new tab', + onclick: (e: Event) => { + e.stopPropagation(); + openInNewTab(); + } + } + ] + : []), + { + icon: conversation.pinned ? PinOff : Pin, + label: conversation.pinned ? 'Unpin' : 'Pin', + onclick: (e: Event) => { + e.stopPropagation(); + handleTogglePin(); + }, + separator: canOpenInNewTab + }, + { + icon: Pencil, + label: 'Edit', + onclick: handleEdit, + shortcut: isActive ? SHORTCUT_EDIT_CONVERSATION : undefined + }, + { + icon: Download, + label: 'Export', + onclick: (e: Event) => { + e.stopPropagation(); + conversationsStore.downloadConversation(conversation.id); + }, + shortcut: isActive ? SHORTCUT_EXPORT_CONVERSATION : undefined + }, + { + icon: ListChecks, + label: 'Select', + onclick: handleEnterSelectionMode + }, + { + icon: Trash2, + label: 'Delete', + onclick: handleDelete, + separator: true, + shortcut: isActive ? SHORTCUT_DELETE_CONVERSATION : undefined, + variant: 'destructive' + } + ]); + function handleTogglePin() { conversationsStore.toggleConversationPin(conversation.id); } @@ -178,154 +234,108 @@ }); - + + {#if !isSelectionMode && renderActionsDropdown} +
+ +
+ {/if} + + diff --git a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationActions.svelte b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationActions.svelte index dc5f25a6f42d..6dccbb088dca 100644 --- a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationActions.svelte +++ b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationActions.svelte @@ -12,6 +12,7 @@ import { SidebarAction } from '$lib/enums'; import { deviceStore } from '$lib/stores'; import type { DesktopIconStripItem } from '$lib/types'; + import { hapticTap } from '$lib/utils'; import type { Component } from 'svelte'; import { onMount } from 'svelte'; import { circIn } from 'svelte/easing'; @@ -103,7 +104,12 @@ ? 'bg-accent text-accent-foreground' : ''}" href={item.route} - onclick={itemOnClick} + onclick={() => { + // a phone confirms the tap by feel, as a native list does + if (deviceStore.isMobile) hapticTap(); + + itemOnClick?.(); + }} size="default" variant="ghost" > diff --git a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte index 9b329c92a1e7..e804ed91e4d7 100644 --- a/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte +++ b/tools/ui/src/lib/components/app/navigation/SidebarNavigation/SidebarNavigationConversationItem.svelte @@ -19,21 +19,25 @@ import { FORK_TREE_DEPTH_PADDING, ICON_CLASS_DEFAULT, + ROW_LONG_PRESS_MS, SHORTCUT_DELETE_CONVERSATION, SHORTCUT_EDIT_CONVERSATION, SHORTCUT_EXPORT_CONVERSATION, UI_DATA_ATTRS } from '$lib/constants'; + import { useLongPress } from '$lib/hooks/use-long-press.svelte'; import { RouterService } from '$lib/services/router.service'; import { chatStore, conversationsStore, + deviceStore, settingsStore, sidebarDragStore, tabsStore } from '$lib/stores'; import type { MenuAction } from '$lib/types'; - import { onMount } from 'svelte'; + import { hapticTap } from '$lib/utils'; + import { onMount, tick } from 'svelte'; interface Props { isActive?: boolean; @@ -70,6 +74,20 @@ let renderActionsDropdown = $state(false); let dropdownOpen = $state(false); let contextMenuOpen = $state(false); + let rowElement = $state(null); + + // a phone has no three-dot trigger: holding the row opens its menu, anchored to the row + const longPress = useLongPress({ + enabled: () => deviceStore.isMobile && !isSelectionMode, + onLongPress: openHoldMenu + }); + + // the hold itself taps the haptic engine + async function openHoldMenu() { + renderActionsDropdown = true; + await tick(); + dropdownOpen = true; + } let isLoading = $derived(chatStore.getAllLoadingChats().includes(conversation.id)); let isLanding = $derived(sidebarDragStore.landingConvId === conversation.id); @@ -205,6 +223,8 @@ } else if ((event.metaKey || event.ctrlKey) && canOpenInNewTab) { openInNewTab(); } else { + if (deviceStore.isMobile) hapticTap(); + onSelect?.(conversation.id); } } @@ -256,16 +276,24 @@