diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..d289e43042e2 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4087,6 +4087,43 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + static const bool fuse_attn_set = ggml_cuda_env_int("GGML_CUDA_FUSE_ATTN_SET", 1) != 0; + if (fuse_attn_set && node->op == GGML_OP_FLASH_ATTN_EXT && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const ggml_op ops[] = { GGML_OP_FLASH_ATTN_EXT, GGML_OP_RESHAPE, GGML_OP_SET }; + const int output_idx = i + 2; + if (ggml_can_fuse_subgraph(cgraph, i, 3, ops, &output_idx, 1)) { + ggml_tensor * view = cgraph->nodes[i + 1]; + ggml_tensor * set = cgraph->nodes[output_idx]; + const int32_t * params = (const int32_t *) set->op_params; + const size_t offset = (size_t) params[3]; + if (view->src[0] == node && set->src[1] == view && params[4] == 1 && + node->type == GGML_TYPE_F32 && set->type == GGML_TYPE_F32 && + ggml_is_contiguous(node) && ggml_is_contiguous(set) && + view->ne[2] == 1 && view->ne[3] == 1 && params[0] == view->nb[1] && + offset <= ggml_nbytes(set) && ggml_nbytes(node) <= ggml_nbytes(set) - offset && + ggml_cuda_flash_attn_ext_get_alloc_size(cuda_ctx->device, node) == ggml_nbytes(node)) { + ggml_tensor dst = *node; + dst.data = (char *) set->data + offset; + dst.buffer = set->buffer; + bool overlap = false; + const uintptr_t begin = (uintptr_t) dst.data; + const uintptr_t end = begin + ggml_nbytes(&dst); + for (int j = 0; j < GGML_MAX_SRC; ++j) { + const ggml_tensor * src = node->src[j]; + if (src) { + const uintptr_t src_begin = (uintptr_t) src->data; + overlap = overlap || (begin < src_begin + ggml_nbytes(src) && src_begin < end); + } + } + if (!overlap && begin % 16 == 0) { + ggml_cuda_flash_attn_ext(*cuda_ctx, &dst); + return 2; + } + } + } + } + + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/scripts/hip/gcn-cdna-vgpr-check.py b/scripts/hip/gcn-cdna-vgpr-check.py index 2d371c691986..04a06ef76b0b 100644 --- a/scripts/hip/gcn-cdna-vgpr-check.py +++ b/scripts/hip/gcn-cdna-vgpr-check.py @@ -156,27 +156,28 @@ def main(): '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_S4_S4_i', '_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', '_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', + # KV_rows_ptr added in 1b6935b22 changes the HIP names of these existing exceptions. + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiS4_PfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil', '_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', - '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', + '_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil', } functions = parse_log_file(log_file) diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..a44ee04f3baa 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -8926,6 +8926,23 @@ struct test_flash_attn_ext_kv_rows : public test_case { } }; +struct test_flash_attn_ext_set : public test_flash_attn_ext { + const bool extra_consumer, inplace; + test_flash_attn_ext_set(int64_t hs, int64_t tokens, bool extra_consumer = false, bool inplace = true) + : test_flash_attn_ext(hs, hs, 4, {1, 2}, tokens, tokens, false), extra_consumer(extra_consumer), inplace(inplace) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return "FLASH_ATTN_EXT_SET"; } + std::string vars() override { return test_flash_attn_ext::vars() + "," + VARS_TO_STR2(extra_consumer, inplace); } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * attn = test_flash_attn_ext::build_graph(ctx); + ggml_tensor * flat = ggml_reshape_2d(ctx, attn, attn->ne[0] * attn->ne[1], attn->ne[2] * attn->ne[3]); + ggml_tensor * buffer = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, flat->ne[0], flat->ne[1] + 3); + ggml_tensor * out = inplace ? ggml_set_2d_inplace(ctx, buffer, flat, buffer->nb[1], buffer->nb[1]) : + ggml_set_2d(ctx, buffer, flat, buffer->nb[1], buffer->nb[1]); + return extra_consumer ? ggml_add(ctx, out, ggml_sum(ctx, flat)) : out; + } +}; + // GGML_OP_CROSS_ENTROPY_LOSS struct test_cross_entropy_loss : public test_case { const ggml_type type; @@ -12554,6 +12571,14 @@ static std::vector> make_test_cases_eval() { } // asymmetric head_dim (hsk != hsv) with one or both sides not 64-aligned + for (int64_t hs : {80, 128}) { + for (int64_t tokens : {16, 64, 65}) { + test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens)); + test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens, true)); + test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens, false, false)); + } + } + test_cases.emplace_back(new test_flash_attn_ext(72, 64, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 72, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 113, 75, true, true, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); diff --git a/tools/mtmd/clip.cpp b/tools/mtmd/clip.cpp index 9a043d7f9ca3..ca70568b4ccb 100644 --- a/tools/mtmd/clip.cpp +++ b/tools/mtmd/clip.cpp @@ -162,6 +162,8 @@ struct clip_ctx { int max_nodes = 8192; ggml_backend_sched_ptr sched; + ggml_backend_sched_eval_callback cb_eval; + void * cb_eval_user_data; clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO; bool is_allocated = false; @@ -174,7 +176,8 @@ struct clip_ctx { bool support_batch = false; - clip_ctx(clip_context_params & ctx_params) { + clip_ctx(clip_context_params & ctx_params) : + cb_eval(ctx_params.cb_eval), cb_eval_user_data(ctx_params.cb_eval_user_data) { flash_attn_type = ctx_params.flash_attn_type; no_alloc = ctx_params.no_alloc; backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr); @@ -237,6 +240,38 @@ struct clip_ctx { } }; +static void clip_ensure_graph_capacity(clip_ctx * ctx, const clip_image_f32 & img) { + static const bool window_batch = [] { + const char * value = std::getenv("MTMD_QWEN25VL_WINDOW_BATCH"); + return value == nullptr || std::atoi(value) != 0; + }(); + const auto & hparams = ctx->model.hparams; + if (window_batch && ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL && hparams.n_wa_pattern > 0 && + ctx->flash_attn_type != CLIP_FLASH_ATTN_TYPE_DISABLED) { + const int patch_size = hparams.patch_size; + const int n_patches_y = img.ny() / patch_size; + const int window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; + const int grid_window = window_size / patch_size / 2; + GGML_ASSERT(grid_window > 0); + // Each row of windows has at most two runs of equal-length windows. + const int64_t max_runs = 2 * ((int64_t(n_patches_y) / 2 + grid_window - 1) / grid_window); + const int64_t max_nodes = 8192 + int64_t(hparams.n_layer) * max_runs * 32; + if (max_nodes > std::numeric_limits::max()) { + throw std::runtime_error("Qwen2.5-VL window graph is too large"); + } + if (max_nodes > ctx->max_nodes) { + ctx->max_nodes = int(max_nodes); + ctx->sched.reset(ggml_backend_sched_new(ctx->backend_ptrs.data(), ctx->backend_buft.data(), + ctx->backend_ptrs.size(), ctx->max_nodes, false, true)); + ctx->is_allocated = false; + if (ctx->cb_eval != nullptr) { + ggml_backend_sched_set_eval_callback(ctx->sched.get(), ctx->cb_eval, ctx->cb_eval_user_data); + } + ctx->buf_compute_meta.resize(ctx->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(ctx->max_nodes, false)); + } + } +} + // // clip_graph // @@ -259,6 +294,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) : eps(hparams.eps), kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f), flash_attn_type(ctx->flash_attn_type) { + clip_ensure_graph_capacity(ctx, img); struct ggml_init_params params = { /*.mem_size =*/ ctx->buf_compute_meta.size(), /*.mem_buffer =*/ ctx->buf_compute_meta.data(), @@ -813,8 +849,12 @@ ggml_tensor * clip_graph::build_attn( if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) { ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3); - k = ggml_cast(ctx0, k, GGML_TYPE_F16); - v = ggml_cast(ctx0, v, GGML_TYPE_F16); + if (k->type != GGML_TYPE_F16) { + k = ggml_cast(ctx0, k, GGML_TYPE_F16); + } + if (v->type != GGML_TYPE_F16) { + v = ggml_cast(ctx0, v, GGML_TYPE_F16); + } // mask must be f16 here, use build_inp_attn_mask() GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16); @@ -5254,6 +5294,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { return false; } + // Grow before the warmup check so a replacement scheduler gets its buffers reserved. + clip_ensure_graph_capacity(ctx, imgs.entries[0]); + // if buffers are not allocated, we need to do a warmup run to allocate them if (!ctx->is_allocated) { clip_model_loader::warmup(*ctx, *params->imgs); @@ -5839,7 +5882,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { const int grid_window = attn_window_size / patch_size / merge_ratio; int dst = 0; // [num_vision_tokens, num_vision_tokens] attention mask tensor - std::vector mask(pow(ipw * iph, 2), std::numeric_limits::lowest()); + const bool has_mask = ggml_graph_get_tensor(gf, get_attn_mask_name(0).c_str()) != nullptr; + std::vector mask(has_mask ? size_t(ipw * iph) * (ipw * iph) : 0, std::numeric_limits::lowest()); int mask_row = 0; for (int y = 0; y < ph; y += grid_window) { @@ -5859,7 +5903,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { } } - for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) { + for (int r=0; has_mask && r < win_h * win_w * merge_ratio * merge_ratio; r++) { int row_offset = mask_row * (ipw * iph); std::fill( mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio), @@ -5872,7 +5916,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) { set_input_i32("window_idx", idx); set_input_i32("inv_window_idx", inv_idx); - set_input_attn_mask(mask); + if (has_mask) { + set_input_attn_mask(mask); + } } else { for (int i = 0; i < ph * pw; i++) { idx[i] = i; diff --git a/tools/mtmd/models/qwen2vl.cpp b/tools/mtmd/models/qwen2vl.cpp index 7516aa23a986..126827ff6287 100644 --- a/tools/mtmd/models/qwen2vl.cpp +++ b/tools/mtmd/models/qwen2vl.cpp @@ -1,15 +1,47 @@ #include "models.h" +#include + ggml_cgraph * clip_graph_qwen2vl::build() { GGML_ASSERT(model.patch_bias == nullptr); GGML_ASSERT(model.class_embedding == nullptr); const int batch_size = 1; const bool use_window_attn = hparams.n_wa_pattern > 0; + static const bool window_batch = [] { + const char * value = std::getenv("MTMD_QWEN25VL_WINDOW_BATCH"); + return value == nullptr || std::atoi(value) != 0; + }(); + static const bool window_cast = [] { + const char * value = std::getenv("MTMD_QWEN25VL_KV_CAST_EARLY"); + return value == nullptr || std::atoi(value) != 0; + }(); + const bool batch_windows = window_batch && use_window_attn && flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED; const int n_wa_pattern = hparams.n_wa_pattern; const int n_pos = n_patches; const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position + // Keep the input window order and batch adjacent windows of equal size. + std::vector> window_runs; + if (batch_windows) { + const int merge_ratio = 2; + const int window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112; + const int grid_window = window_size / patch_size / merge_ratio; + GGML_ASSERT(grid_window > 0); + const int pw = n_patches_x / merge_ratio; + const int ph = n_patches_y / merge_ratio; + for (int y = 0; y < ph; y += grid_window) { + for (int x = 0; x < pw; x += grid_window) { + const int len = std::min(grid_window, ph - y) * std::min(grid_window, pw - x) * merge_ratio * merge_ratio; + if (!window_runs.empty() && window_runs.back().first == len) { + window_runs.back().second++; + } else { + window_runs.emplace_back(len, 1); + } + } + } + } + norm_type norm_t = proj_type == PROJECTOR_TYPE_QWEN25VL ? NORM_TYPE_RMS // qwen 2.5 vl : NORM_TYPE_NORMAL; // qwen 2 vl @@ -52,8 +84,9 @@ ggml_cgraph * clip_graph_qwen2vl::build() { inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4); ggml_set_name(inv_window_idx, "inv_window_idx"); ggml_set_input(inv_window_idx); - // mask for window attention - window_mask = build_inp_attn_mask(n_pos, n_pos); + if (!batch_windows) { + window_mask = build_inp_attn_mask(n_pos, n_pos); + } // inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size] GGML_ASSERT(batch_size == 1); @@ -103,8 +136,42 @@ ggml_cgraph * clip_graph_qwen2vl::build() { ggml_tensor * attn_mask = full_attn ? nullptr : window_mask; - cur = build_attn(layer.o_w, layer.o_b, - Qcur, Kcur, Vcur, attn_mask, kq_scale, il); + if (batch_windows && !full_attn) { + // Convert each full K/V tensor once before taking window views. + if (window_cast) { + Kcur = ggml_cast(ctx0, Kcur, GGML_TYPE_F16); + Vcur = ggml_cast(ctx0, Vcur, GGML_TYPE_F16); + } + std::vector outputs; + int offset = 0; + for (const auto & run : window_runs) { + auto view = [&](ggml_tensor * t) { + return ggml_view_4d(ctx0, t, d_head, n_head, run.first, run.second, + t->nb[1], t->nb[2], t->nb[2] * run.first, t->nb[2] * offset); + }; + outputs.push_back(build_attn(nullptr, nullptr, + view(Qcur), view(Kcur), view(Vcur), nullptr, kq_scale, il)); + offset += run.first * run.second; + } + GGML_ASSERT(offset == n_patches); + ggml_tensor * joined = outputs.front(); + if (outputs.size() > 1) { + joined = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_patches); + size_t offset_bytes = 0; + for (ggml_tensor * output : outputs) { + joined = ggml_set_2d_inplace(ctx0, joined, output, joined->nb[1], offset_bytes); + offset_bytes += ggml_nbytes(output); + } + GGML_ASSERT(offset_bytes == ggml_nbytes(joined)); + } + cur = build_mm(layer.o_w, joined); + if (layer.o_b) { + cur = ggml_add(ctx0, cur, layer.o_b); + } + } else { + cur = build_attn(layer.o_w, layer.o_b, + Qcur, Kcur, Vcur, attn_mask, kq_scale, il); + } cb(cur, "attn_out", il); }