From db4a8fe81f9b450149c55fcbd68a48898701cd54 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 19:17:23 -0700 Subject: [PATCH] hip: fuse two-axis RoPE and enable D512 WMMA attention Keep the existing grid loops for oversized tensors and specialize one-row-per-block vision RoPE. Enable the existing D512 4x8 tile on RDNA3.5 with FP16 KV and no logit softcap. Assisted-by: Codex --- ggml/src/ggml-cuda/fattn-mma-f16.cuh | 15 ++++- ggml/src/ggml-cuda/fattn.cu | 14 ++++ ggml/src/ggml-cuda/ggml-cuda.cu | 29 +++++++++ ggml/src/ggml-cuda/rope.cu | 97 ++++++++++++++++++++-------- ggml/src/ggml-cuda/rope.cuh | 2 +- tests/test-backend-ops.cpp | 36 +++++++++-- 6 files changed, 159 insertions(+), 34 deletions(-) diff --git a/ggml/src/ggml-cuda/fattn-mma-f16.cuh b/ggml/src/ggml-cuda/fattn-mma-f16.cuh index 96b596ea6fb5..0dafc8713c8d 100644 --- a/ggml/src/ggml-cuda/fattn-mma-f16.cuh +++ b/ggml/src/ggml-cuda/fattn-mma-f16.cuh @@ -322,10 +322,18 @@ static constexpr __device__ int ggml_cuda_fattn_mma_get_nstages_target(const int } static __host__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols, const int cc) { + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && DKQ == 512 && DV == 512 && ncols == 32) { + return false; + } return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols, cc).Q_in_reg; } static constexpr __device__ bool ggml_cuda_fattn_mma_get_Q_in_reg(const int DKQ, const int DV, const int ncols) { +#if defined(RDNA3_5) + if (DKQ == 512 && DV == 512 && ncols == 32) { + return false; + } +#endif return ggml_cuda_fattn_mma_get_config(DKQ, DV, ncols).Q_in_reg; } @@ -1866,7 +1874,12 @@ static __global__ void flash_attn_ext_f16( #endif // __CUDA_ARCH__ == GGML_CUDA_CC_TURING #if defined(AMD_WMMA_AVAILABLE) - if (ncols1*ncols2 < 16 || ncols2 == 1 || DKQ > 256) { +#if defined(RDNA3_5) + constexpr bool language_d512 = DKQ == 512 && DV == 512 && ncols1 == 4 && ncols2 == 8; +#else + constexpr bool language_d512 = false; +#endif + if (ncols1*ncols2 < 16 || ncols2 == 1 || (DKQ > 256 && !language_d512)) { NO_DEVICE_CODE; return; } diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu index 5d7ebf966378..8a7f78555c07 100644 --- a/ggml/src/ggml-cuda/fattn.cu +++ b/ggml/src/ggml-cuda/fattn.cu @@ -732,6 +732,20 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const } } + static const bool wmma_d512 = []() { + const char * env = getenv("GGML_CUDA_FATTN_WMMA_D512"); + return env == nullptr || atoi(env) != 0; + }(); + float logit_softcap = 0.0f; + memcpy(&logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); + // D512 uses the existing 4 x 8 tile, which fits RDNA shared memory. + if (wmma_d512 && + GGML_CUDA_CC_IS_RDNA3_5(cc) && amd_wmma_available(cc) && gqa_opt_applies && + Q->ne[0] == 512 && V->ne[0] == 512 && gqa_ratio == 8 && Q->ne[1] >= 32 && + logit_softcap == 0.0f && K->type == GGML_TYPE_F16 && V->type == GGML_TYPE_F16) { + return BEST_FATTN_KERNEL_MMA_F16; + } + // AMD WMMA is faster than the tile kernel if the wide tiles with high arithmetic intensity can be utilized. if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 256) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > (Q->ne[0] <= 128 ? 8 : 16)) { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..a3dceddd9730 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4087,6 +4087,35 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + static const bool fuse_norm_rope_2axis = []() { + const char * env = getenv("GGML_CUDA_FUSE_NORM_ROPE_2AXIS"); + return !env || atoi(env) != 0; + }(); + if (fuse_norm_rope_2axis && node->op == GGML_OP_RMS_NORM && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const ggml_op ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_ROPE, GGML_OP_ROPE }; + const int output_idx = i + 3; + if (ggml_can_fuse_subgraph(cgraph, i, 4, ops, &output_idx, 1)) { + ggml_tensor * mul = cgraph->nodes[i + 1]; + ggml_tensor * first = cgraph->nodes[i + 2]; + ggml_tensor * second = cgraph->nodes[output_idx]; + const int32_t * fp = (const int32_t *) first->op_params; + const int32_t * sp = (const int32_t *) second->op_params; + if (ggml_cuda_should_fuse_rms_norm_mul_rope(node, mul, first) && + (mul->src[0] == node || mul->src[1] == node) && + first->src[2] == nullptr && second->src[2] == nullptr && second->src[0] == first && + fp[2] == GGML_ROPE_TYPE_NEOX && fp[1] > 0 && 2*fp[1] == node->ne[0] && node->ne[0] < 1024 && + sp[15] == fp[1] && memcmp(fp, sp, 15*sizeof(int32_t)) == 0 && + second->type == GGML_TYPE_F32 && ggml_are_same_shape(first, second) && ggml_is_contiguous(second) && + second->src[1]->type == GGML_TYPE_I32 && ggml_is_contiguous(second->src[1]) && + ggml_nelements(second->src[1]) >= node->ne[2] && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 4, &output_idx, 1)) { + ggml_cuda_op_rms_norm_mul_rope_fused(*cuda_ctx, node, mul, first, nullptr, second); + return 3; + } + } + } + + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index 45e813406597..17f47dfa0c13 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -726,7 +726,7 @@ void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rope // fused RMS_NORM + MUL + ROPE (+ VIEW + SET_ROWS) // one block per row: block_reduce gives the norm scale, then each thread applies mul and rope to the elements it owns -template +template static __global__ void rms_norm_mul_rope_f32( const float * x, D * dst, const int ncols, const int nchannels, const int nsamples, const int64_t s01, const int64_t s02, const int64_t s03, @@ -741,7 +741,7 @@ static __global__ void rms_norm_mul_rope_f32( const rope_corr_dims corr_dims, const float theta_scale, const float * freq_factors, const int64_t * row_indices, const int set_rows_stride, - const bool is_neox) { + const bool is_neox, const int32_t * pos_second) { ggml_cuda_pdl_lc(); const int row = blockIdx.x; const int tid = threadIdx.x; @@ -751,8 +751,8 @@ static __global__ void rms_norm_mul_rope_f32( ggml_cuda_pdl_sync(); // grid.y and grid.z are clamped to the CUDA limit, iterate over the excess channels/samples - for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { - for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { + for (int sample = blockIdx.z; !loop_grid || sample < nsamples; sample += gridDim.z) { + for (int channel = blockIdx.y; !loop_grid || channel < nchannels; channel += gridDim.y) { const float * xc = x + sample*s03 + channel*s02 + row*s01; const uint32_t mul_row = fastmodulo(row, mul_nrows_packed); @@ -778,40 +778,57 @@ static __global__ void rms_norm_mul_rope_f32( D * dstc = dst + idst; for (int i0 = 2*tid; i0 < ncols; i0 += 2*block_size) { + const int offset = two_axis && i0 >= n_dims ? n_dims : 0; + const int ir = i0 - offset; int ix0; int ix1; - if (is_neox && i0 < n_dims) { - ix0 = i0/2; - ix1 = i0/2 + n_dims/2; + if (is_neox && ir < n_dims) { + ix0 = offset + ir/2; + ix1 = offset + ir/2 + n_dims/2; } else { ix0 = i0 + 0; ix1 = i0 + 1; } - const float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)]; - const float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)]; + float x0 = scale * xc[ix0] * mulc[fastmodulo(ix0, mul_ncols_packed)]; + float x1 = scale * xc[ix1] * mulc[fastmodulo(ix1, mul_ncols_packed)]; + if constexpr (two_axis) { + if (offset != 0) { + #if defined(GGML_USE_HIP) + asm volatile("" : "+v"(x0), "+v"(x1)); + #else + asm volatile("" : "+f"(x0), "+f"(x1)); + #endif + } + } - if (i0 >= n_dims) { + if (ir >= n_dims) { dstc[ix0] = ggml_cuda_cast(x0); dstc[ix1] = ggml_cuda_cast(x1); continue; } - const float theta_base = pos[channel]*powf(theta_scale, i0/2.0f); - const float freq_factor = has_ff ? freq_factors[i0/2] : 1.0f; + const float theta_base = (two_axis && offset != 0 ? pos_second[channel] : pos[channel])*powf(theta_scale, ir/2.0f); + const float freq_factor = has_ff ? freq_factors[ir/2] : 1.0f; float cos_theta; float sin_theta; - rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, i0, ext_factor, attn_factor, cos_theta, sin_theta); + rope_yarn(theta_base/freq_factor, freq_scale, corr_dims, ir, ext_factor, attn_factor, cos_theta, sin_theta); dstc[ix0] = ggml_cuda_cast(x0*cos_theta - x1*sin_theta); dstc[ix1] = ggml_cuda_cast(x0*sin_theta + x1*cos_theta); } - if constexpr (block_size > WARP_SIZE) { + if constexpr (block_size > WARP_SIZE && loop_grid) { // sync is needed as we reuse s_sum across block_reduce invocations, see #26385 __syncthreads(); } + if constexpr (!loop_grid) { + break; + } + } + if constexpr (!loop_grid) { + break; } } } @@ -832,7 +849,7 @@ static void rms_norm_mul_rope_cuda( const rope_corr_dims corr_dims, const float * freq_factors, const int64_t * row_indices, const int set_rows_stride, - const bool is_neox, cudaStream_t stream) { + const bool is_neox, cudaStream_t stream, const int32_t * pos_second = nullptr) { GGML_ASSERT(ncols % 2 == 0); const dim3 blocks_num(nrows, MIN(nchannels, UINT16_MAX), MIN(nsamples, UINT16_MAX)); @@ -844,6 +861,27 @@ static void rms_norm_mul_rope_cuda( const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); + if (pos_second != nullptr) { + static const bool norm_rope_small_block = []() { + const char * env = getenv("GGML_CUDA_NORM_ROPE_SMALL_BLOCK"); + return !env || atoi(env) != 0; + }(); + GGML_ASSERT(ncols < 1024 && ncols == 2*n_dims && is_neox && !freq_factors && !set_rows_stride); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const bool small_block = norm_rope_small_block && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols == 128 && nchannels >= 128; + const ggml_cuda_kernel_launch_params params = {blocks_num, dim3(small_block ? 128 : 256), 32*sizeof(float), stream}; + const bool loop_grid = nchannels > UINT16_MAX || nsamples > UINT16_MAX; + const auto kernel = small_block ? + (loop_grid ? rms_norm_mul_rope_f32<128, false, D, true> : rms_norm_mul_rope_f32<128, false, D, true, false>) : + (loop_grid ? rms_norm_mul_rope_f32<256, false, D, true> : rms_norm_mul_rope_f32<256, false, D, true, false>); + ggml_cuda_kernel_launch(kernel, params, + x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, + mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, + n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, + freq_factors, row_indices, set_rows_stride, is_neox, pos_second); + return; + } + if (ncols < 1024) { const dim3 block_dims(256, 1, 1); const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; @@ -852,13 +890,13 @@ static void rms_norm_mul_rope_cuda( x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, - freq_factors, row_indices, set_rows_stride, is_neox); + freq_factors, row_indices, set_rows_stride, is_neox, nullptr); } else { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params, x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, - freq_factors, row_indices, set_rows_stride, is_neox); + freq_factors, row_indices, set_rows_stride, is_neox, nullptr); } } else { const dim3 block_dims(1024, 1, 1); @@ -868,19 +906,19 @@ static void rms_norm_mul_rope_cuda( x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, - freq_factors, row_indices, set_rows_stride, is_neox); + freq_factors, row_indices, set_rows_stride, is_neox, nullptr); } else { ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<1024, true, D>, launch_params, x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, - freq_factors, row_indices, set_rows_stride, is_neox); + freq_factors, row_indices, set_rows_stride, is_neox, nullptr); } } } void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, - ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows) { + ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows, ggml_tensor * rope_second) { const ggml_tensor * x = rms_norm->src[0]; const ggml_tensor * mul_src = mul->src[0] == rms_norm ? mul->src[1] : mul->src[0]; @@ -892,8 +930,9 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, GGML_ASSERT(mul_src->type == GGML_TYPE_F32); GGML_ASSERT(rope->type == GGML_TYPE_F32); - void * dst_d = rope->data; - ggml_type dst_type = rope->type; + const ggml_tensor * output = rope_second ? rope_second : rope; + void * dst_d = output->data; + ggml_type dst_type = output->type; const int64_t * row_indices = nullptr; int set_rows_stride = 0; @@ -943,10 +982,10 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, const int64_t mul_s02 = mul_src->nb[2] / ts_mul; const int64_t mul_s03 = mul_src->nb[3] / ts_mul; - const size_t ts_dst = ggml_type_size(rope->type); - const int64_t s1 = rope->nb[1] / ts_dst; - const int64_t s2 = rope->nb[2] / ts_dst; - const int64_t s3 = rope->nb[3] / ts_dst; + const size_t ts_dst = ggml_type_size(output->type); + const int64_t s1 = output->nb[1] / ts_dst; + const int64_t s2 = output->nb[2] / ts_dst; + const int64_t s3 = output->nb[3] / ts_dst; cudaStream_t stream = ctx.stream(); @@ -956,14 +995,16 @@ void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, (const float *) mul_src->data, mul_s01, mul_s02, mul_s03, mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3], n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, - freq_factors, row_indices, set_rows_stride, is_neox, stream); + freq_factors, row_indices, set_rows_stride, is_neox, stream, + rope_second ? (const int32_t *) rope_second->src[1]->data : nullptr); } else if (dst_type == GGML_TYPE_F16) { rms_norm_mul_rope_cuda((const float *) x->data, (half *) dst_d, x->ne[0], x->ne[1], x->ne[2], x->ne[3], s01, s02, s03, s1, s2, s3, eps, (const float *) mul_src->data, mul_s01, mul_s02, mul_s03, mul_src->ne[0], mul_src->ne[1], mul_src->ne[2], mul_src->ne[3], n_dims, pos, freq_scale, freq_base, ext_factor, attn_factor, corr_dims, - freq_factors, row_indices, set_rows_stride, is_neox, stream); + freq_factors, row_indices, set_rows_stride, is_neox, stream, + rope_second ? (const int32_t *) rope_second->src[1]->data : nullptr); } else { GGML_ABORT("fatal error"); } diff --git a/ggml/src/ggml-cuda/rope.cuh b/ggml/src/ggml-cuda/rope.cuh index 7ce2d71c508e..35db1195f970 100644 --- a/ggml/src/ggml-cuda/rope.cuh +++ b/ggml/src/ggml-cuda/rope.cuh @@ -8,4 +8,4 @@ void ggml_cuda_op_rope_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows); -void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows); +void ggml_cuda_op_rms_norm_mul_rope_fused(ggml_backend_cuda_context & ctx, ggml_tensor * rms_norm, ggml_tensor * mul, ggml_tensor * rope, ggml_tensor * set_rows, ggml_tensor * rope_second = nullptr); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..55b4ff609cbd 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2952,6 +2952,7 @@ struct test_rms_norm_mul_rope : public test_case { const bool broadcast; // multiply by a 1D [ne0] weight, as model norm weights are const ggml_type set_rows_type; int mode; + const bool two_axis, extra_consumer; std::string op_desc(ggml_tensor * t) override { GGML_UNUSED(t); @@ -2961,14 +2962,14 @@ struct test_rms_norm_mul_rope : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR9(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type); + return VARS_TO_STR9(ne, eps, multi_add, mul, rope, set_rows, broadcast, mode, set_rows_type) + "," + VARS_TO_STR2(two_axis, extra_consumer); } test_rms_norm_mul_rope(std::array ne, float eps = 1e-6f, bool multi_add = false, bool set_rows = false, bool broadcast = false, int mode = GGML_ROPE_TYPE_NORMAL, - bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16) + bool mul = true, bool rope = true, ggml_type set_rows_type = GGML_TYPE_F16, bool two_axis = false, bool extra_consumer = false) : ne(ne), eps(eps), multi_add(multi_add), mul(mul), rope(rope), set_rows(set_rows), broadcast(broadcast), - set_rows_type(set_rows_type), mode(mode) {} + set_rows_type(set_rows_type), mode(mode), two_axis(two_axis), extra_consumer(extra_consumer) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, ne[0], ne[1], ne[2], ne[3]); @@ -3006,7 +3007,15 @@ struct test_rms_norm_mul_rope : public test_case { int sections[4] = { n_dims/3, n_dims/3, n_dims/3, 0 }; a = ggml_rope_multi(ctx, a, pos, nullptr, n_dims, sections, mode, 0, 10000.0f, 1.0f, 0.0f, 1.0f, 32.0f, 1.0f); } else { - a = ggml_rope(ctx, a, pos, ne[0], mode); + a = ggml_rope(ctx, a, pos, two_axis ? ne[0]/2 : ne[0], mode); + if (two_axis) { + ggml_tensor * first = a; + ggml_tensor * pos_y = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, ne[2]); + a = ggml_rope_set_offset(ggml_rope(ctx, a, pos_y, ne[0]/2, mode), ne[0]/2); + if (extra_consumer) { + a = ggml_add(ctx, a, first); + } + } } } @@ -8814,6 +8823,12 @@ struct test_flash_attn_ext : public test_case { } }; +struct test_d512_wmma : public test_flash_attn_ext { + test_d512_wmma(int queries, int batches, bool mask) + : test_flash_attn_ext(512, 512, 2, {8, batches}, 512, queries, mask) {} + std::string op_desc(ggml_tensor *) override { return "D512_WMMA"; } +}; + // large Q values, so the online softmax has to rescale the partial results struct test_flash_attn_ext_large_logits : public test_flash_attn_ext { static constexpr int q_range = 20; @@ -10337,6 +10352,19 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + + for (int width : {64, 128, 256}) { + for (bool extra : {false, true}) { + test_cases.emplace_back(new test_rms_norm_mul_rope({width, 4, 129, 1}, 1e-6f, false, false, true, + GGML_ROPE_TYPE_NEOX, true, true, GGML_TYPE_F16, true, extra)); + } + } + test_cases.emplace_back(new test_rms_norm_mul_rope({64, 1, 65536, 1}, 1e-6f, false, false, true, + GGML_ROPE_TYPE_NEOX, true, true, GGML_TYPE_F16, true, false)); + for (int queries : {31, 32, 64, 129}) { + test_cases.emplace_back(new test_d512_wmma(queries, 1, true)); + test_cases.emplace_back(new test_d512_wmma(queries, 2, false)); + } std::default_random_engine rng(0); // unary ops