Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions ggml/src/ggml-cuda/ggml-cuda.cu
Original file line number Diff line number Diff line change
Expand Up @@ -4087,6 +4087,43 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph

ggml_tensor * node = cgraph->nodes[i];

static const bool fuse_attn_set = ggml_cuda_env_int("GGML_CUDA_FUSE_ATTN_SET", 1) != 0;
if (fuse_attn_set && node->op == GGML_OP_FLASH_ATTN_EXT && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) {
const ggml_op ops[] = { GGML_OP_FLASH_ATTN_EXT, GGML_OP_RESHAPE, GGML_OP_SET };
const int output_idx = i + 2;
if (ggml_can_fuse_subgraph(cgraph, i, 3, ops, &output_idx, 1)) {
ggml_tensor * view = cgraph->nodes[i + 1];
ggml_tensor * set = cgraph->nodes[output_idx];
const int32_t * params = (const int32_t *) set->op_params;
const size_t offset = (size_t) params[3];
if (view->src[0] == node && set->src[1] == view && params[4] == 1 &&
node->type == GGML_TYPE_F32 && set->type == GGML_TYPE_F32 &&
ggml_is_contiguous(node) && ggml_is_contiguous(set) &&
view->ne[2] == 1 && view->ne[3] == 1 && params[0] == view->nb[1] &&
offset <= ggml_nbytes(set) && ggml_nbytes(node) <= ggml_nbytes(set) - offset &&
ggml_cuda_flash_attn_ext_get_alloc_size(cuda_ctx->device, node) == ggml_nbytes(node)) {
ggml_tensor dst = *node;
dst.data = (char *) set->data + offset;
dst.buffer = set->buffer;
bool overlap = false;
const uintptr_t begin = (uintptr_t) dst.data;
const uintptr_t end = begin + ggml_nbytes(&dst);
for (int j = 0; j < GGML_MAX_SRC; ++j) {
const ggml_tensor * src = node->src[j];
if (src) {
const uintptr_t src_begin = (uintptr_t) src->data;
overlap = overlap || (begin < src_begin + ggml_nbytes(src) && src_begin < end);
}
}
if (!overlap && begin % 16 == 0) {
ggml_cuda_flash_attn_ext(*cuda_ctx, &dst);
return 2;
}
}
}
}


if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() &&
ggml_cuda_match_shared_expert(cgraph, i, i + 3) &&
ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) {
Expand Down
41 changes: 21 additions & 20 deletions scripts/hip/gcn-cdna-vgpr-check.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,27 +156,28 @@ def main():
'_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_S4_S4_i',
'_ZL9mul_mat_qIL9ggml_type10ELi64ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_',
'_ZL9mul_mat_qIL9ggml_type42ELi128ELb1EEvPKcPKiS4_S4_PfS5_PKf15HIP_vector_typeIjLj3EEiiiiiS9_S9_iiiS9_S9_iiiS9_',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil',
# KV_rows_ptr added in 1b6935b22 changes the HIP names of these existing exceptions.
'_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi32ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi16ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi4ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi16ELi2ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi4ELi8ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi512ELi512ELi8ELi4ELb1ELb0ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi32ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi2ELi16ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi8ELi4ELb0ELb1ELb0EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_vecILi256ELi1EL9ggml_type2ELS0_2ELb0EEvPKcS2_S2_S2_S2_PKiS4_PfP15HIP_vector_typeIfLj2EEffffjfiS6_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi320ELi256ELi2ELi32ELb0ELb0ELb0EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiPfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
'_ZL18flash_attn_ext_f16ILi576ELi512ELi1ELi16ELb0ELb1ELb1EEvPKcS1_S1_S1_S1_PKiS3_PfP15HIP_vector_typeIfLj2EEffffjfiS5_IjLj3EEiiiiiiiiiiiliiliiiiil',
}

functions = parse_log_file(log_file)
Expand Down
25 changes: 25 additions & 0 deletions tests/test-backend-ops.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8926,6 +8926,23 @@ struct test_flash_attn_ext_kv_rows : public test_case {
}
};

struct test_flash_attn_ext_set : public test_flash_attn_ext {
const bool extra_consumer, inplace;
test_flash_attn_ext_set(int64_t hs, int64_t tokens, bool extra_consumer = false, bool inplace = true)
: test_flash_attn_ext(hs, hs, 4, {1, 2}, tokens, tokens, false), extra_consumer(extra_consumer), inplace(inplace) {}
bool run_whole_graph() override { return true; }
std::string op_desc(ggml_tensor *) override { return "FLASH_ATTN_EXT_SET"; }
std::string vars() override { return test_flash_attn_ext::vars() + "," + VARS_TO_STR2(extra_consumer, inplace); }
ggml_tensor * build_graph(ggml_context * ctx) override {
ggml_tensor * attn = test_flash_attn_ext::build_graph(ctx);
ggml_tensor * flat = ggml_reshape_2d(ctx, attn, attn->ne[0] * attn->ne[1], attn->ne[2] * attn->ne[3]);
ggml_tensor * buffer = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, flat->ne[0], flat->ne[1] + 3);
ggml_tensor * out = inplace ? ggml_set_2d_inplace(ctx, buffer, flat, buffer->nb[1], buffer->nb[1]) :
ggml_set_2d(ctx, buffer, flat, buffer->nb[1], buffer->nb[1]);
return extra_consumer ? ggml_add(ctx, out, ggml_sum(ctx, flat)) : out;
}
};

// GGML_OP_CROSS_ENTROPY_LOSS
struct test_cross_entropy_loss : public test_case {
const ggml_type type;
Expand Down Expand Up @@ -12554,6 +12571,14 @@ static std::vector<std::unique_ptr<test_case>> make_test_cases_eval() {
}

// asymmetric head_dim (hsk != hsv) with one or both sides not 64-aligned
for (int64_t hs : {80, 128}) {
for (int64_t tokens : {16, 64, 65}) {
test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens));
test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens, true));
test_cases.emplace_back(new test_flash_attn_ext_set(hs, tokens, false, false));
}
}

test_cases.emplace_back(new test_flash_attn_ext(72, 64, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(64, 72, 4, {1, 1}, 256, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
test_cases.emplace_back(new test_flash_attn_ext(65, 67, 4, {1, 1}, 113, 75, true, true, 8.0f, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16));
Expand Down
58 changes: 52 additions & 6 deletions tools/mtmd/clip.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,8 @@ struct clip_ctx {

int max_nodes = 8192;
ggml_backend_sched_ptr sched;
ggml_backend_sched_eval_callback cb_eval;
void * cb_eval_user_data;
clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO;
bool is_allocated = false;

Expand All @@ -174,7 +176,8 @@ struct clip_ctx {

bool support_batch = false;

clip_ctx(clip_context_params & ctx_params) {
clip_ctx(clip_context_params & ctx_params) :
cb_eval(ctx_params.cb_eval), cb_eval_user_data(ctx_params.cb_eval_user_data) {
flash_attn_type = ctx_params.flash_attn_type;
no_alloc = ctx_params.no_alloc;
backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);
Expand Down Expand Up @@ -237,6 +240,38 @@ struct clip_ctx {
}
};

static void clip_ensure_graph_capacity(clip_ctx * ctx, const clip_image_f32 & img) {
static const bool window_batch = [] {
const char * value = std::getenv("MTMD_QWEN25VL_WINDOW_BATCH");
return value == nullptr || std::atoi(value) != 0;
}();
const auto & hparams = ctx->model.hparams;
if (window_batch && ctx->proj_type() == PROJECTOR_TYPE_QWEN25VL && hparams.n_wa_pattern > 0 &&
ctx->flash_attn_type != CLIP_FLASH_ATTN_TYPE_DISABLED) {
const int patch_size = hparams.patch_size;
const int n_patches_y = img.ny() / patch_size;
const int window_size = hparams.attn_window_size > 0 ? hparams.attn_window_size : 112;
const int grid_window = window_size / patch_size / 2;
GGML_ASSERT(grid_window > 0);
// Each row of windows has at most two runs of equal-length windows.
const int64_t max_runs = 2 * ((int64_t(n_patches_y) / 2 + grid_window - 1) / grid_window);
const int64_t max_nodes = 8192 + int64_t(hparams.n_layer) * max_runs * 32;
if (max_nodes > std::numeric_limits<int>::max()) {
throw std::runtime_error("Qwen2.5-VL window graph is too large");
}
if (max_nodes > ctx->max_nodes) {
ctx->max_nodes = int(max_nodes);
ctx->sched.reset(ggml_backend_sched_new(ctx->backend_ptrs.data(), ctx->backend_buft.data(),
Comment thread
jimw567 marked this conversation as resolved.
Resolved
ctx->backend_ptrs.size(), ctx->max_nodes, false, true));
ctx->is_allocated = false;
if (ctx->cb_eval != nullptr) {
ggml_backend_sched_set_eval_callback(ctx->sched.get(), ctx->cb_eval, ctx->cb_eval_user_data);
}
ctx->buf_compute_meta.resize(ctx->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead_custom(ctx->max_nodes, false));
}
}
}

//
// clip_graph
//
Expand All @@ -259,6 +294,7 @@ clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :
eps(hparams.eps),
kq_scale(d_head > 0 ? 1.0f / sqrtf((float)d_head) : 0.0f),
flash_attn_type(ctx->flash_attn_type) {
clip_ensure_graph_capacity(ctx, img);
struct ggml_init_params params = {
/*.mem_size =*/ ctx->buf_compute_meta.size(),
/*.mem_buffer =*/ ctx->buf_compute_meta.data(),
Expand Down Expand Up @@ -813,8 +849,12 @@ ggml_tensor * clip_graph::build_attn(
if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {
ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3);

k = ggml_cast(ctx0, k, GGML_TYPE_F16);
v = ggml_cast(ctx0, v, GGML_TYPE_F16);
if (k->type != GGML_TYPE_F16) {
k = ggml_cast(ctx0, k, GGML_TYPE_F16);
}
if (v->type != GGML_TYPE_F16) {
v = ggml_cast(ctx0, v, GGML_TYPE_F16);
}
// mask must be f16 here, use build_inp_attn_mask()
GGML_ASSERT(!kq_mask || kq_mask->type == GGML_TYPE_F16);

Expand Down Expand Up @@ -5254,6 +5294,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
return false;
}

// Grow before the warmup check so a replacement scheduler gets its buffers reserved.
clip_ensure_graph_capacity(ctx, imgs.entries[0]);

// if buffers are not allocated, we need to do a warmup run to allocate them
if (!ctx->is_allocated) {
clip_model_loader::warmup(*ctx, *params->imgs);
Expand Down Expand Up @@ -5839,7 +5882,8 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
const int grid_window = attn_window_size / patch_size / merge_ratio;
int dst = 0;
// [num_vision_tokens, num_vision_tokens] attention mask tensor
std::vector<float> mask(pow(ipw * iph, 2), std::numeric_limits<float>::lowest());
const bool has_mask = ggml_graph_get_tensor(gf, get_attn_mask_name(0).c_str()) != nullptr;
std::vector<float> mask(has_mask ? size_t(ipw * iph) * (ipw * iph) : 0, std::numeric_limits<float>::lowest());
int mask_row = 0;

for (int y = 0; y < ph; y += grid_window) {
Expand All @@ -5859,7 +5903,7 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
}
}

for (int r=0; r < win_h * win_w * merge_ratio * merge_ratio; r++) {
for (int r=0; has_mask && r < win_h * win_w * merge_ratio * merge_ratio; r++) {
int row_offset = mask_row * (ipw * iph);
std::fill(
mask.begin() + row_offset + (dst_0 * merge_ratio * merge_ratio),
Expand All @@ -5872,7 +5916,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {

set_input_i32("window_idx", idx);
set_input_i32("inv_window_idx", inv_idx);
set_input_attn_mask(mask);
if (has_mask) {
set_input_attn_mask(mask);
}
} else {
for (int i = 0; i < ph * pw; i++) {
idx[i] = i;
Expand Down
Loading
Loading