diff --git a/ggml/src/ggml-cuda/cpy.cu b/ggml/src/ggml-cuda/cpy.cu index 90a06b97bbc6..fe9accd878c9 100644 --- a/ggml/src/ggml-cuda/cpy.cu +++ b/ggml/src/ggml-cuda/cpy.cu @@ -1,3 +1,4 @@ +#include "opt-traffic.cuh" #include "cpy.cuh" #include "dequantize.cuh" #include "cpy-utils.cuh" @@ -11,6 +12,29 @@ const int CUDA_CPY_TILE_DIM_2D = 32; // 2D tile dimension for transposed blocks const int CUDA_CPY_BLOCK_NM = 8; // block size of 3rd dimension if available const int CUDA_CPY_BLOCK_ROWS = 8; // block dimension for marching through rows +static __global__ void cpy_f32_f16_vec8( + const char * src, uint4 * dst, const uint32_t n, const uint3 ne0, const uint3 ne1, const uint3 ne2, + const size_t nb1, const size_t nb2, const size_t nb3) { + ggml_cuda_pdl_lc(); + const uint32_t i = blockIdx.x*blockDim.x + threadIdx.x; + if (i >= n) { + return; + } + const uint32_t c = fastmodulo(i, ne0); + const uint32_t row = fastdiv(i, ne0); + const uint32_t r1 = fastmodulo(row, ne1); + const uint32_t r23 = fastdiv(row, ne1); + const uint32_t r2 = fastmodulo(r23, ne2); + const uint32_t r3 = fastdiv(r23, ne2); + ggml_cuda_pdl_sync(); + const float4 * ptr = (const float4 *) (src + 32*size_t(c) + nb1*r1 + nb2*r2 + nb3*r3); + const float4 a = ptr[0], b = ptr[1]; + alignas(16) const half values[8] = {ggml_cuda_cast(a.x), ggml_cuda_cast(a.y), + ggml_cuda_cast(a.z), ggml_cuda_cast(a.w), ggml_cuda_cast(b.x), + ggml_cuda_cast(b.y), ggml_cuda_cast(b.z), ggml_cuda_cast(b.w)}; + ggml_cuda_memcpy_1<16>(dst + i, values); +} + template static __global__ void cpy_scalar(const char * cx, char * cdst, const int64_t ne, const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t nb00, const int64_t nb01, const int64_t nb02, @@ -494,7 +518,19 @@ void ggml_cuda_cpy(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, gg (src0_ddc, src1_ddc, ne, ne00, ne01, ne02, nb00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb13, main_stream); } } else if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F16) { - if (contiguous_srcs) { + if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_CPY_F32_F16_VEC8, true) && + GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[ctx.device].cc) && !contiguous_srcs && + ggml_is_contiguous(src1) && ggml_are_same_shape(src0, src1) && nb00 == sizeof(float) && + ne00 % 8 == 0 && ne >= 8192 && ne/8 <= INT32_MAX && + ne00/8 <= UINT32_MAX && ne01 <= UINT32_MAX && ne02 <= UINT32_MAX && + (uintptr_t) src0_ddc % 16 == 0 && (uintptr_t) src1_ddc % 16 == 0 && + nb01 % 16 == 0 && nb02 % 16 == 0 && nb03 % 16 == 0 && + ((uintptr_t) src0_ddc <= (uintptr_t) src1_ddc ? + (uintptr_t) src1_ddc - (uintptr_t) src0_ddc >= ggml_nbytes(src0) : + (uintptr_t) src0_ddc - (uintptr_t) src1_ddc >= ggml_nbytes(src1))) { + cpy_f32_f16_vec8<<<(ne/8 + 255)/256, 256, 0, main_stream>>>(src0_ddc, (uint4 *) src1_ddc, + ne/8, init_fastdiv_values(ne00/8), init_fastdiv_values(ne01), init_fastdiv_values(ne02), nb01, nb02, nb03); + } else if (contiguous_srcs) { ggml_cpy_scalar_contiguous_cuda (src0_ddc, src1_ddc, ne, main_stream); } else { diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..acf4458f32c3 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1,3 +1,4 @@ +#include "opt-traffic.cuh" #include "ggml-cuda.h" #include "ggml-impl.h" #include "ggml-backend-impl.h" @@ -4087,6 +4088,28 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_FUSE_NORM_MUL_ADD, true) && node->op == GGML_OP_NORM && GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[cuda_ctx->device].cc)) { + const ggml_op ops[] = { GGML_OP_NORM, GGML_OP_MUL, GGML_OP_ADD }; + const int output_idx = i + 2; + if (ggml_can_fuse_subgraph(cgraph, i, 3, ops, &output_idx, 1)) { + ggml_tensor * mul = cgraph->nodes[i + 1]; + ggml_tensor * add = cgraph->nodes[output_idx]; + const ggml_tensor * weight = mul->src[mul->src[0] == node ? 1 : 0]; + const ggml_tensor * bias = add->src[add->src[0] == mul ? 1 : 0]; + if ((mul->src[0] == node || mul->src[1] == node) && (add->src[0] == mul || add->src[1] == mul) && + node->src[0]->type == GGML_TYPE_F32 && weight->type == GGML_TYPE_F32 && bias->type == GGML_TYPE_F32 && + node->type == GGML_TYPE_F32 && mul->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && + ggml_are_same_shape(node, add) && ggml_are_same_shape(node, mul) && + ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(add) && + ggml_is_contiguous(weight) && weight->ne[0] == node->ne[0] && ggml_nrows(weight) == 1 && + ggml_is_contiguous(bias) && bias->ne[0] == node->ne[0] && ggml_nrows(bias) == 1 && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 3, &output_idx, 1)) { + ggml_cuda_op_norm_fused(*cuda_ctx, node, mul, add); + return 2; + } + } + } + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/ggml/src/ggml-cuda/moe-weighted-reduction.cu b/ggml/src/ggml-cuda/moe-weighted-reduction.cu index 11ec58497f1e..8142840c71e4 100644 --- a/ggml/src/ggml-cuda/moe-weighted-reduction.cu +++ b/ggml/src/ggml-cuda/moe-weighted-reduction.cu @@ -24,6 +24,47 @@ static __global__ void moe_weighted_reduction_f32(const float * __restrict__ exp dst[token * n_embd + col] = sum; } +// same sums as moe_weighted_reduction_f32, with all loads of a thread issued before the sum and 4 columns per thread +template +static __global__ void moe_weighted_reduction_f32_x4(const float4 * __restrict__ experts, + const float * __restrict__ expert_scale, + const float * __restrict__ weights, + float4 * __restrict__ dst, + const int64_t n_embd4) { + const int64_t token = blockIdx.x; + const int64_t col = (int64_t) blockIdx.y * blockDim.x + threadIdx.x; + if (col >= n_embd4) { + return; + } + + const uint64_t first_row = (uint64_t) token * n_expert_used; + float4 x[n_expert_used]; + float w[n_expert_used]; +#pragma unroll + for (int expert = 0; expert < n_expert_used; ++expert) { + x[expert] = experts[(first_row + expert) * n_embd4 + col]; + w[expert] = weights[first_row + expert]; + } + float s[n_expert_used]; +#pragma unroll + for (int expert = 0; expert < n_expert_used; ++expert) { + s[expert] = expert_scale != nullptr ? expert_scale[first_row + expert] : 1.0f; + } + float4 sum; + sum.x = (x[0].x * s[0]) * w[0]; + sum.y = (x[0].y * s[0]) * w[0]; + sum.z = (x[0].z * s[0]) * w[0]; + sum.w = (x[0].w * s[0]) * w[0]; +#pragma unroll + for (int expert = 1; expert < n_expert_used; ++expert) { + sum.x += (x[expert].x * s[expert]) * w[expert]; + sum.y += (x[expert].y * s[expert]) * w[expert]; + sum.z += (x[expert].z * s[expert]) * w[expert]; + sum.w += (x[expert].w * s[expert]) * w[expert]; + } + dst[token * n_embd4 + col] = sum; +} + static void launch_moe_weighted_reduction(const float * experts, const float * expert_scale, const float * weights, @@ -32,6 +73,20 @@ static void launch_moe_weighted_reduction(const float * experts, int64_t n_tokens, int n_expert_used, cudaStream_t stream) { + // opt-in: one kernel per common expert count, the generic kernel is latency bound with large n_tokens + static const bool use_x4 = getenv("GGML_CUDA_MOE_REDUCE_X4") != nullptr && atoi(getenv("GGML_CUDA_MOE_REDUCE_X4")) != 0; + if (use_x4 && n_embd % 4 == 0 && (n_expert_used == 8 || n_expert_used == 4) && + (uintptr_t) experts % 16 == 0 && (uintptr_t) dst % 16 == 0) { + constexpr int threads = 128; + const int64_t n_embd4 = n_embd / 4; + const dim3 blocks(n_tokens, (n_embd4 + threads - 1) / threads, 1); + if (n_expert_used == 8) { + moe_weighted_reduction_f32_x4<8><<>>((const float4 *) experts, expert_scale, weights, (float4 *) dst, n_embd4); + } else { + moe_weighted_reduction_f32_x4<4><<>>((const float4 *) experts, expert_scale, weights, (float4 *) dst, n_embd4); + } + return; + } constexpr int threads = 256; const dim3 blocks(n_tokens, (n_embd + threads - 1) / threads, 1); moe_weighted_reduction_f32 diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 3f0ea6f2a474..2d9fbd30264d 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -1,10 +1,11 @@ #include "norm.cuh" #include -template +template static __global__ void norm_f32( const float * x, float * dst, const int ncols, const int nchannels, const int nsamples, - const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps) { + const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, + const float * weight = nullptr, const float * bias = nullptr) { const int nrows = gridDim.x; const int row = blockIdx.x; const int tid = threadIdx.x; @@ -35,7 +36,22 @@ static __global__ void norm_f32( const float inv_std = rsqrtf(var + eps); for (int col = tid; col < ncols; col += block_size) { - dstc[col] = (xc[col] - mean) * inv_std; + float value = (xc[col] - mean) * inv_std; + if constexpr (affine) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(value)); +#else + asm volatile("" : "+f"(value)); +#endif + value *= weight[col]; +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(value)); +#else + asm volatile("" : "+f"(value)); +#endif + value += bias[col]; + } + dstc[col] = value; } if constexpr (block_size > WARP_SIZE) { @@ -569,6 +585,23 @@ void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { norm_f32_cuda(src0_d, dst_d, ne00, ne01, ne02, ne03, s01, s02, s03, eps, stream); } +void ggml_cuda_op_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * norm, ggml_tensor * mul, ggml_tensor * add) { + const ggml_tensor * x = norm->src[0]; + const ggml_tensor * w = mul->src[mul->src[0] == norm ? 1 : 0]; + const ggml_tensor * b = add->src[add->src[0] == mul ? 1 : 0]; + const dim3 grid(x->ne[1], MIN(x->ne[2], int64_t(UINT16_MAX)), MIN(x->ne[3], int64_t(UINT16_MAX))); + const float eps = ggml_get_op_params_f32(norm, 0); + if (x->ne[0] < 1024) { + norm_f32<<>>((const float *) x->data, (float *) add->data, + x->ne[0], x->ne[2], x->ne[3], x->nb[1]/sizeof(float), x->nb[2]/sizeof(float), x->nb[3]/sizeof(float), eps, + (const float *) w->data, (const float *) b->data); + } else { + norm_f32<1024, true><<>>((const float *) x->data, (float *) add->data, + x->ne[0], x->ne[2], x->ne[3], x->nb[1]/sizeof(float), x->nb[2]/sizeof(float), x->nb[3]/sizeof(float), eps, + (const float *) w->data, (const float *) b->data); + } +} + void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; const float * src0_d = (const float *)src0->data; diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index 95618df5fd4d..d04808dcde03 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -18,3 +18,5 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +void ggml_cuda_op_norm_fused(ggml_backend_cuda_context & ctx, ggml_tensor * norm, ggml_tensor * mul, ggml_tensor * add); diff --git a/ggml/src/ggml-cuda/opt-traffic.cuh b/ggml/src/ggml-cuda/opt-traffic.cuh new file mode 100644 index 000000000000..7f16eaa4e099 --- /dev/null +++ b/ggml/src/ggml-cuda/opt-traffic.cuh @@ -0,0 +1,6 @@ +#pragma once + +#include + +#define GGML_CUDA_TRAFFIC_OPT_ENABLED(name, fallback) \ + ([] { static const bool enabled = [] { const char * v = std::getenv(#name); return v ? std::atoi(v) != 0 : (fallback); }(); return enabled; }()) diff --git a/ggml/src/ggml-cuda/rope.cu b/ggml/src/ggml-cuda/rope.cu index 45e813406597..10dc7e69addf 100644 --- a/ggml/src/ggml-cuda/rope.cu +++ b/ggml/src/ggml-cuda/rope.cu @@ -1,3 +1,4 @@ +#include "opt-traffic.cuh" #include "convert.cuh" #include "ggml-cuda/common.cuh" #include "ggml.h" @@ -391,8 +392,14 @@ static void rope_norm_cuda(const T * x, const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); - const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); - const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE); + // opt-in (GGML_CUDA_ROPE_BLOCK): smaller blocks for short rows (head dim 64/128 leaves most of a 256-thread block idle) + static const int block_size = []() { + const char * env = getenv("GGML_CUDA_ROPE_BLOCK"); + const int bs = env ? atoi(env) : 0; + return bs >= 32 && bs <= CUDA_ROPE_BLOCK_SIZE ? bs : CUDA_ROPE_BLOCK_SIZE; + }(); + const dim3 block_dims(1, block_size, 1); + const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size); const dim3 block_nums(nr, n_blocks_x, 1); const float theta_scale = powf(freq_base, -2.0f / n_dims); @@ -435,8 +442,23 @@ static void rope_neox_cuda(const T * x, const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); - const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); - const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE); + int block_size = CUDA_ROPE_BLOCK_SIZE; +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && ne00 == 128) { + block_size = 64; + } +#endif + static const int block_override = []() { + const char * env = getenv("GGML_CUDA_ROPE_BLOCK"); + const int bs = env ? atoi(env) : 0; + return bs >= 32 && bs <= CUDA_ROPE_BLOCK_SIZE ? bs : 0; + }(); + if (block_override) { + block_size = block_override; + } + const dim3 block_dims(1, block_size, 1); + const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size); const dim3 block_nums(nr, n_blocks_x, 1); const float theta_scale = powf(freq_base, -2.0f / n_dims); @@ -482,8 +504,15 @@ static void rope_multi_cuda(const T * x, const bool inplace, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); - const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); - const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE); + int block_size = CUDA_ROPE_BLOCK_SIZE; +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && (ne00 == 72 || ne00 == 80 || ne00 == 128)) { + block_size = 64; + } +#endif + const dim3 block_dims(1, block_size, 1); + const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size); const dim3 block_nums(nr, n_blocks_x, 1); const float theta_scale = powf(freq_base, -2.0f / n_dims); @@ -525,8 +554,15 @@ static void rope_vision_cuda(const T * x, const mrope_sections sections, cudaStream_t stream) { GGML_ASSERT(ne00 % 2 == 0); - const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1); - const int n_blocks_x = (ne00 + 2 * CUDA_ROPE_BLOCK_SIZE - 1) / (2 * CUDA_ROPE_BLOCK_SIZE); + int block_size = CUDA_ROPE_BLOCK_SIZE; +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_TRAFFIC_OPT_ENABLED(GGML_CUDA_ROPE_SMALL_BLOCK, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && (ne00 == 72 || ne00 == 80 || ne00 == 128)) { + block_size = 64; + } +#endif + const dim3 block_dims(1, block_size, 1); + const int n_blocks_x = (ne00 + 2 * block_size - 1) / (2 * block_size); const dim3 block_nums(nr, n_blocks_x, 1); // break down (head_dim, heads, seq) into (CUDA_ROPE_BLOCK_SIZE, x, heads * seq) // where x ~= ceil(head_dim / CUDA_ROPE_BLOCK_SIZE); @@ -844,17 +880,27 @@ static void rms_norm_mul_rope_cuda( const uint3 mul_nchannels_packed = init_fastdiv_values(mul_nchannels); const uint3 mul_nsamples_packed = init_fastdiv_values(mul_nsamples); + static const bool norm_rope_warp = []() { + const char * env = getenv("GGML_CUDA_NORM_ROPE_WARP"); + return env != nullptr && atoi(env) != 0; + }(); if (ncols < 1024) { - const dim3 block_dims(256, 1, 1); + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + const bool one_warp = norm_rope_warp && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols <= 256; + const dim3 block_dims(one_warp ? WARP_SIZE : 256, 1, 1); const ggml_cuda_kernel_launch_params launch_params = {blocks_num, block_dims, 32*sizeof(float), stream}; if (freq_factors == nullptr) { - ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, false, D>, launch_params, + const auto kernel = one_warp ? rms_norm_mul_rope_f32 : + rms_norm_mul_rope_f32<256, false, D>; + ggml_cuda_kernel_launch(kernel, launch_params, x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, freq_factors, row_indices, set_rows_stride, is_neox); } else { - ggml_cuda_kernel_launch(rms_norm_mul_rope_f32<256, true, D>, launch_params, + const auto kernel = one_warp ? rms_norm_mul_rope_f32 : + rms_norm_mul_rope_f32<256, true, D>; + ggml_cuda_kernel_launch(kernel, launch_params, x, dst, ncols, nchannels, nsamples, s01, s02, s03, s1, s2, s3, eps, mul, mul_s01, mul_s02, mul_s03, mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, n_dims, pos, freq_scale, ext_factor, attn_factor, corr_dims, theta_scale, diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..a36903f64d46 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -3892,21 +3892,23 @@ struct test_norm_mul_add : public test_case { bool run_whole_graph() override { return true; } std::string vars() override { - return VARS_TO_STR4(type, ne, eps, broadcast); + return VARS_TO_STR4(type, ne, eps, broadcast) + "," + VARS_TO_STR2(vector_affine, extra_consumer); } + const bool vector_affine, extra_consumer; + test_norm_mul_add(ggml_type type = GGML_TYPE_F32, std::array ne = {128, 2, 1, 1}, float eps = 1e-5f, - bool broadcast = false) - : type(type), ne(ne), eps(eps), broadcast(broadcast) {} + bool broadcast = false, bool vector_affine = false, bool extra_consumer = false) + : type(type), ne(ne), eps(eps), broadcast(broadcast), vector_affine(vector_affine), extra_consumer(extra_consumer) {} ggml_tensor * build_graph(ggml_context * ctx) override { std::array broadcast_dims = {ne[0], ne[1] * 2, ne[2] * 2, ne[3] * 2}; ggml_tensor * a = ggml_new_tensor(ctx, type, 4, broadcast ? broadcast_dims.data() : ne.data()); - ggml_tensor * w = ggml_new_tensor(ctx, type, 4, ne.data()); - ggml_tensor * b = ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * w = vector_affine ? ggml_new_tensor_1d(ctx, type, ne[0]) : ggml_new_tensor(ctx, type, 4, ne.data()); + ggml_tensor * b = vector_affine ? ggml_new_tensor_1d(ctx, type, ne[0]) : ggml_new_tensor(ctx, type, 4, ne.data()); ggml_set_param(a); ggml_set_param(w); ggml_set_param(b); ggml_set_name(a, "a"); ggml_set_name(w, "w"); ggml_set_name(b, "b"); @@ -3917,7 +3919,7 @@ struct test_norm_mul_add : public test_case { ggml_tensor * m = ggml_mul(ctx, n, w); ggml_tensor * out = ggml_add(ctx, m, b); ggml_set_name(out, "out"); - return out; + return extra_consumer ? ggml_add(ctx, out, n) : out; } }; // GGML_OP_NORM/RMS_NORM + GGML_OP_SCALE @@ -10337,6 +10339,11 @@ static void add_rdna35_mmq_cases(std::vector> & test_ // Test cases for evaluation: should try to cover edge cases while using small input sizes to keep the runtime low static std::vector> make_test_cases_eval() { std::vector> test_cases; + for (int64_t width : {768, 1023, 1024, 1152, 1280}) { + for (bool extra : {false, true}) { + test_cases.emplace_back(new test_norm_mul_add(GGML_TYPE_F32, {width, 65, 1, 1}, 1e-5f, false, true, extra)); + } + } std::default_random_engine rng(0); // unary ops