From 76818b19a6176235780cbc596fd07e5a20f3c0b9 Mon Sep 17 00:00:00 2001 From: lichang Date: Thu, 12 Nov 2026 13:45:12 -0700 Subject: [PATCH] ggml-cuda: fuse 16-bit BLAS producers and outputs on RDNA3.5 Keep compatible FP16/BF16 FFN intermediates in pooled BLAS buffers and combine conversion with bias, residual, GLU/GELU, RMSNorm, and clamp work. Preserve GEMM precision, FP32 rounding boundaries, existing dispatch fallbacks, and gfx11 normalization/output-stride fixes. Include runtime family and component switches plus graph tests for precision overrides, extra consumers, mixed types, odd sizes, and the normalized, plain, gated, and clamped FFN patterns. Assisted-by: Codex --- ggml/src/ggml-cuda/common.cuh | 10 + ggml/src/ggml-cuda/convert.cu | 152 ++++++++++++ ggml/src/ggml-cuda/convert.cuh | 7 + ggml/src/ggml-cuda/ggml-cuda.cu | 417 +++++++++++++++++++++++++++++++- ggml/src/ggml-cuda/norm.cu | 52 +++- ggml/src/ggml-cuda/norm.cuh | 3 + ggml/src/ggml-cuda/unary.cu | 180 +++++++++++++- ggml/src/ggml-cuda/unary.cuh | 13 + tests/test-backend-ops.cpp | 218 +++++++++++++++++ 9 files changed, 1028 insertions(+), 24 deletions(-) diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index 4d7f945673ac..43b3a45cbaec 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -43,6 +43,16 @@ #define STRINGIZE_IMPL(...) #__VA_ARGS__ #define STRINGIZE(...) STRINGIZE_IMPL(__VA_ARGS__) +// Cache each runtime switch once; explicit 0 restores the fallback. +#define GGML_CUDA_OPT_ENABLED(name, default_value) \ + ([] { \ + static const bool enabled = [] { \ + const char * value = std::getenv(#name); \ + return value ? std::atoi(value) != 0 : (default_value); \ + }(); \ + return enabled; \ + }()) + #define WARP_SIZE 32 #define CUDART_HMAX 11070 // CUDA 11.7, min. ver. for which __hmax and __hmax2 are known to work (may be higher than needed) #define CUDART_HMASK 12000 // CUDA 12.0, min. ver. for half2 -> uint mask comparisons diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 3dc9ef3c43eb..a6d0e5f0209d 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -528,6 +528,158 @@ static void convert_unary_cuda(const void * vx, dst_t * y, (vx, y, ne00, ne01, ne0203, ne02_fdv, s01, s02, s03); } +template +struct alignas(2*sizeof(T)) convert_pair { + T v[2]; +}; + +template +static __device__ __forceinline__ dst_t convert_unary_value(src_t x, float min, float max) { + if constexpr (clamp) { + float value = ggml_cuda_cast(x); + if constexpr (std::is_same_v) { + // Preserve the separate conversion's NaN behavior before the clamp. +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(value)); +#else + asm volatile("" : "+f"(value)); +#endif + } + return ggml_cuda_cast(fminf(fmaxf(value, min), max)); + } + return ggml_cuda_cast(x); +} + +template +static __global__ void convert_unary_packed(const src_t * __restrict__ x, dst_t * __restrict__ y, const int64_t k, + const float min = 0.0f, const float max = 0.0f) { + const int64_t i = 2*(int64_t(blockIdx.x)*blockDim.x + threadIdx.x); + if (i + 1 < k) { + const auto in = *reinterpret_cast *>(x + i); + const convert_pair out = {{convert_unary_value(in.v[0], min, max), + convert_unary_value(in.v[1], min, max)}}; + *reinterpret_cast *>(y + i) = out; + } else if (i < k) { + y[i] = convert_unary_value(x[i], min, max); + } +} + +template +static __global__ void convert_unary_clamped(const src_t * x, dst_t * y, int64_t k, float min, float max) { + const int64_t i = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (i < k) { + y[i] = convert_unary_value(x[i], min, max); + } +} + +template +static __global__ void convert_bias_to_f32(const T * x, const float * bias, float * y, int64_t ncols, int64_t nrows, const float * residual) { + const int64_t col = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (col >= ncols) { + return; + } + const float b = bias[col]; + for (int64_t row = blockIdx.y; row < nrows; row += gridDim.y) { + const int64_t i = row*ncols + col; + float v = ggml_cuda_cast(x[i]) + b; + if constexpr (add_residual) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(v)); +#else + asm volatile("" : "+f"(v)); +#endif + v += residual[i]; + } + y[i] = v; + } +} + + +template +static __global__ void convert_bias_to_f32_vec2(const convert_pair * x, const float2 * bias, float2 * y, + int64_t ncols, int64_t nrows, const float2 * residual) { + const int64_t i = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (i >= ncols*nrows) { + return; + } + const float2 b = bias[i % ncols]; + { + const auto in = x[i]; + float2 v = make_float2(ggml_cuda_cast(in.v[0]) + b.x, ggml_cuda_cast(in.v[1]) + b.y); + if constexpr (add_residual) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(v.x), "+v"(v.y)); +#else + asm volatile("" : "+f"(v.x), "+f"(v.y)); +#endif + const float2 r = residual[i]; + v.x += r.x; + v.y += r.y; + } + y[i] = v; + } +} + +template +void ggml_cuda_convert_bias_to_f32(const T * x, const float * bias, float * y, int64_t ncols, int64_t nrows, cudaStream_t stream, const float * residual) { +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_BIAS_VEC, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols % 2 == 0 && + reinterpret_cast(x) % alignof(convert_pair) == 0 && + reinterpret_cast(bias) % alignof(float2) == 0 && + reinterpret_cast(y) % alignof(float2) == 0 && + (!residual || reinterpret_cast(residual) % alignof(float2) == 0)) { + const dim3 blocks((ncols*nrows/2 + CUDA_DEQUANTIZE_BLOCK_SIZE - 1)/CUDA_DEQUANTIZE_BLOCK_SIZE); + if (residual) { + convert_bias_to_f32_vec2<<>>( + reinterpret_cast *>(x), reinterpret_cast(bias), + reinterpret_cast(y), ncols/2, nrows, reinterpret_cast(residual)); + } else { + convert_bias_to_f32_vec2<<>>( + reinterpret_cast *>(x), reinterpret_cast(bias), + reinterpret_cast(y), ncols/2, nrows, nullptr); + } + return; + } +#endif + const dim3 blocks((ncols + CUDA_DEQUANTIZE_BLOCK_SIZE - 1)/CUDA_DEQUANTIZE_BLOCK_SIZE, std::min(nrows, int64_t(65535))); + if (residual) { + convert_bias_to_f32<<>>(x, bias, y, ncols, nrows, residual); + } else { + convert_bias_to_f32<<>>(x, bias, y, ncols, nrows, nullptr); + } +} + +template +static void convert_unary_clamped_cuda(const src_t * x, dst_t * y, int64_t k, float min, float max, cudaStream_t stream) { + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_CONVERT_CLAMP_VEC2, true) && + reinterpret_cast(x) % alignof(convert_pair) == 0 && + reinterpret_cast(y) % alignof(convert_pair) == 0) { + const int64_t blocks = (k + 2*CUDA_DEQUANTIZE_BLOCK_SIZE - 1)/(2*CUDA_DEQUANTIZE_BLOCK_SIZE); + convert_unary_packed<<>>(x, y, k, min, max); + } else { + const int64_t blocks = (k + CUDA_DEQUANTIZE_BLOCK_SIZE - 1)/CUDA_DEQUANTIZE_BLOCK_SIZE; + convert_unary_clamped<<>>(x, y, k, min, max); + } +} + +template +void ggml_cuda_convert_clamp_to_f32(const T * x, float * y, int64_t k, float min, float max, cudaStream_t stream) { + convert_unary_clamped_cuda(x, y, k, min, max, stream); +} + +template +void ggml_cuda_convert_clamp_from_f32(const float * x, T * y, int64_t k, float min, float max, cudaStream_t stream) { + convert_unary_clamped_cuda(x, y, k, min, max, stream); +} + +template void ggml_cuda_convert_clamp_to_f32(const half *, float *, int64_t, float, float, cudaStream_t); +template void ggml_cuda_convert_clamp_from_f32(const float *, half *, int64_t, float, float, cudaStream_t); +template void ggml_cuda_convert_bias_to_f32(const half *, const float *, float *, int64_t, int64_t, cudaStream_t, const float *); +template void ggml_cuda_convert_clamp_to_f32(const nv_bfloat16 *, float *, int64_t, float, float, cudaStream_t); +template void ggml_cuda_convert_clamp_from_f32(const float *, nv_bfloat16 *, int64_t, float, float, cudaStream_t); +template void ggml_cuda_convert_bias_to_f32(const nv_bfloat16 *, const float *, float *, int64_t, int64_t, cudaStream_t, const float *); + template static void convert_unary_cont_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { if (k % 4 == 0 && diff --git a/ggml/src/ggml-cuda/convert.cuh b/ggml/src/ggml-cuda/convert.cuh index 91255ccf31aa..84d9643f54ec 100644 --- a/ggml/src/ggml-cuda/convert.cuh +++ b/ggml/src/ggml-cuda/convert.cuh @@ -16,6 +16,13 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type); to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type); +template +void ggml_cuda_convert_clamp_to_f32(const T * x, float * y, int64_t k, float min, float max, cudaStream_t stream); +template +void ggml_cuda_convert_clamp_from_f32(const float * x, T * y, int64_t k, float min, float max, cudaStream_t stream); +template +void ggml_cuda_convert_bias_to_f32(const T * x, const float * bias, float * y, int64_t ncols, int64_t nrows, cudaStream_t stream, const float * residual = nullptr); + // TODO more general support for non-contiguous inputs template diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 929353e00ec9..9d31a83daad6 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -1667,8 +1667,15 @@ struct batched_mul_mat_traits { static inline auto convert_nc(ggml_type src_type) { return ggml_get_to_fp16_nc_cuda(src_type); } }; +struct ggml_cuda_cublas_16bit_fusion { + const void * src1 = nullptr; + void * dst = nullptr; + const ggml_tensor * dst_bias = nullptr; + const ggml_tensor * dst_residual = nullptr; +}; + template -static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { +static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const ggml_tensor * dst_clamp = nullptr, const ggml_tensor * src1_clamp = nullptr, const ggml_tensor * src1_glu = nullptr, const ggml_cuda_cublas_16bit_fusion * fusion = nullptr) { using traits = batched_mul_mat_traits; using cuda_t = typename traits::cuda_type; @@ -1679,6 +1686,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const GGML_TENSOR_BINARY_OP_LOCALS const int64_t ldc = nb1 / sizeof(float); + const int64_t ne_dst = ggml_nelements(dst); cudaStream_t main_stream = ctx.stream(); cublasHandle_t cublas_h = ctx.cublas_handle(); @@ -1695,7 +1703,7 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const int64_t s12 = nb12 / src1_ts; int64_t s13 = nb13 / src1_ts; - float * dst_ddf = (float *) dst->data; + float * dst_ddf = (float *) (fusion && fusion->dst_residual ? fusion->dst_residual->data : fusion && fusion->dst_bias ? fusion->dst_bias->data : dst_clamp ? dst_clamp->data : dst->data); const cuda_t * src0_ptr = nullptr; const cuda_t * src1_ptr = nullptr; @@ -1731,7 +1739,13 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const src0_ptr = src0_alloc.get(); } - if (src1->type == compute_type) { + if (fusion && fusion->src1) { + GGML_ASSERT(compute_type != GGML_TYPE_F32 && ggml_is_contiguous(src1)); + src1_ptr = (const cuda_t *) fusion->src1; + s11 = ne10; + s12 = ne11*s11; + s13 = ne12*s12; + } else if (src1->type == compute_type) { src1_ptr = (const cuda_t *) src1->data; } else { src1_alloc.alloc(ggml_nelements(src1)); @@ -1739,7 +1753,19 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const if (ggml_is_contiguously_allocated(src1)) { const auto convert_func = traits::convert(src1->type); GGML_ASSERT(convert_func != nullptr); - convert_func(src1->data, src1_alloc.get(), ggml_nelements(src1), main_stream); + if constexpr (compute_type == GGML_TYPE_F16 || compute_type == GGML_TYPE_BF16) { + if (src1_clamp) { + ggml_cuda_convert_clamp_from_f32((const float *) src1_clamp->src[0]->data, src1_alloc.get(), + ggml_nelements(src1), ggml_get_op_params_f32(src1_clamp, 0), ggml_get_op_params_f32(src1_clamp, 1), main_stream); + } else if (src1_glu) { + ggml_cuda_glu_to_16bit((const float *) src1_glu->src[0]->data, (const float *) src1_glu->src[1]->data, + src1_alloc.get(), ggml_nelements(src1), ggml_get_glu_op(src1_glu), main_stream); + } else { + convert_func(src1->data, src1_alloc.get(), ggml_nelements(src1), main_stream); + } + } else { + convert_func(src1->data, src1_alloc.get(), ggml_nelements(src1), main_stream); + } const size_t src1_bs = ggml_blck_size(src1->type); s11 *= src1_bs; s12 *= src1_bs; @@ -1776,7 +1802,12 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const prefer_f32_output = !GGML_CUDA_CC_IS_RDNA3(cc) && !GGML_CUDA_CC_IS_RDNA4_M(cc) && !GGML_CUDA_CC_IS_CDNA(cc); } - if (prefer_f32_output) { + if (fusion && fusion->dst) { + GGML_ASSERT(compute_type != GGML_TYPE_F32 && !prefer_f32_output && !dst_clamp && !fusion->dst_bias); + dst_ptr = (char *) fusion->dst; + nbd2 /= sizeof(float) / sizeof(cuda_t); + nbd3 /= sizeof(float) / sizeof(cuda_t); + } else if (prefer_f32_output) { dst_ptr = (char *) dst_ddf; cu_compute_type = batched_mul_mat_traits::compute_type; cu_data_type = batched_mul_mat_traits::data_type; @@ -1874,10 +1905,32 @@ static void ggml_cuda_mul_mat_cublas_impl(ggml_backend_cuda_context & ctx, const CUBLAS_GEMM_DEFAULT_TENSOR_OP)); } - // Convert output back to F32 if needed - if (cu_data_type != CUDA_R_32F) { - const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val); - to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream); + // Convert output back to F32 if needed. + if constexpr (compute_type != GGML_TYPE_F32) { + if (cu_data_type != CUDA_R_32F && !(fusion && fusion->dst)) { + if (fusion && fusion->dst_bias) { + GGML_ASSERT(!dst_clamp); + ggml_cuda_convert_bias_to_f32(dst_temp.get(), (const float *) fusion->dst_bias->src[1]->data, + dst_ddf, dst->ne[0], ggml_nrows(dst), main_stream, + fusion->dst_residual ? (const float *) fusion->dst_residual->src[fusion->dst_residual->src[0] == fusion->dst_bias ? 1 : 0]->data : nullptr); + } else if (dst_clamp) { + ggml_cuda_convert_clamp_to_f32(dst_temp.get(), dst_ddf, ne_dst, + ggml_get_op_params_f32(dst_clamp, 0), ggml_get_op_params_f32(dst_clamp, 1), main_stream); + } else { + const to_fp32_cuda_t to_fp32_cuda = ggml_get_to_fp32_cuda(traits::ggml_type_val); + to_fp32_cuda(dst_temp.get(), dst_ddf, ne_dst, main_stream); + } + } + } +} + +static void ggml_cuda_mul_mat_cublas_fused(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, + ggml_tensor * dst, const ggml_tensor * dst_clamp = nullptr, const ggml_tensor * src1_clamp = nullptr, + const ggml_tensor * src1_glu = nullptr, const ggml_cuda_cublas_16bit_fusion * fusion = nullptr) { + if (src0->type == GGML_TYPE_BF16) { + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst, dst_clamp, src1_clamp, src1_glu, fusion); + } else { + ggml_cuda_mul_mat_cublas_impl(ctx, src0, src1, dst, dst_clamp, src1_clamp, src1_glu, fusion); } } @@ -4078,6 +4131,262 @@ static bool ggml_cuda_can_fuse(const struct ggml_cgraph * cgraph, return false; } +static bool ggml_cuda_should_fuse_cublas_16bit(ggml_backend_cuda_context & ctx, const ggml_tensor * mm) { + const auto & device = ggml_cuda_info().devices[ctx.device]; + const ggml_tensor * w = mm->src[0]; + const ggml_tensor * x = mm->src[1]; + if (!GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_16BIT, true) || + (w->type == GGML_TYPE_F16 && !GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_FP16, true)) || + (w->type == GGML_TYPE_BF16 && !GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_BF16, true))) { + return false; + } + return GGML_CUDA_CC_IS_RDNA3_5(device.cc) && (w->type == GGML_TYPE_F16 || w->type == GGML_TYPE_BF16) && x->type == GGML_TYPE_F32 && + mm->type == GGML_TYPE_F32 && ggml_is_contiguous(w) && ggml_is_contiguous(x) && ggml_is_contiguous(mm) && + mm->ne[2] == 1 && mm->ne[3] == 1 && mm->ne[1] >= 32 && + w->ne[2] == 1 && w->ne[3] == 1 && x->ne[2] == 1 && x->ne[3] == 1 && + w->ne[0] > 0 && w->ne[1] > 0 && w->ne[0] <= INT_MAX / w->ne[1] && + x->ne[1] > 0 && x->ne[0] > 0 && x->ne[0] <= INT_MAX / x->ne[1] && + mm->ne[0] > 0 && mm->ne[0] <= INT_MAX / mm->ne[1] && + ggml_get_op_params_i32(mm, 0) == GGML_PREC_DEFAULT && ggml_get_op_params_i32(mm, 1) == 0 && + getenv("GGML_CUDA_CUBLAS_COMPUTE_TYPE") == nullptr && + !ggml_cuda_should_use_mmvf(w->type, device.cc, device.warp_size, w->ne, w->nb, x->ne[1]) && + !ggml_cuda_should_use_mmf(w->type, device.cc, device.warp_size, w->ne, w->nb, x->ne[1], false); +} + +static bool ggml_cuda_safe_elementwise_alias(const ggml_tensor * src, const ggml_tensor * dst) { + const uintptr_t src_begin = (uintptr_t) src->data; + const uintptr_t dst_begin = (uintptr_t) dst->data; + if ((src_begin <= dst_begin && dst_begin - src_begin >= ggml_nbytes(src)) || + (dst_begin <= src_begin && src_begin - dst_begin >= ggml_nbytes(dst))) { + return true; + } + return src_begin == dst_begin && ggml_are_same_shape(src, dst) && ggml_is_contiguous(src); +} + +static bool ggml_cuda_can_fuse_cublas_bias(const ggml_tensor * mm, const ggml_tensor * add) { + const ggml_tensor * bias = add->src[1]; + if (bias) { + const uintptr_t b = (uintptr_t) bias->data; + const uintptr_t d = (uintptr_t) add->data; + if ((b <= d && d - b < ggml_nbytes(bias)) || (d <= b && b - d < ggml_nbytes(add))) { + return false; + } + } + return add->op == GGML_OP_ADD && add->src[0] == mm && bias && bias->type == GGML_TYPE_F32 && + add->type == GGML_TYPE_F32 && ggml_is_contiguous(bias) && ggml_is_contiguous(add) && + ggml_are_same_shape(mm, add) && bias->ne[0] == mm->ne[0] && ggml_nrows(bias) == 1; +} + +static bool ggml_cuda_can_fuse_cublas_residual(const ggml_tensor * bias, const ggml_tensor * add) { + if (!GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_RESIDUAL, true) || + add->op != GGML_OP_ADD || (add->src[0] != bias && add->src[1] != bias)) { + return false; + } + const ggml_tensor * residual = add->src[add->src[0] == bias ? 1 : 0]; + return residual != bias->src[0] && residual->view_src != bias->src[0] && + residual->type == GGML_TYPE_F32 && add->type == GGML_TYPE_F32 && + ggml_are_same_shape(bias, add) && ggml_are_same_shape(residual, add) && + ggml_is_contiguous(residual) && ggml_is_contiguous(add) && + ggml_cuda_safe_elementwise_alias(residual, add) && ggml_cuda_safe_elementwise_alias(bias->src[1], add); +} + +template +static int ggml_cuda_try_fuse_16bit_ffn(ggml_backend_cuda_context & ctx, ggml_cgraph * cgraph, int i, int norm_idx = -1) { + using traits = batched_mul_mat_traits; + using cuda_t = typename traits::cuda_type; + if (!GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_FFN, true) || + (norm_idx >= 0 && !GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_FFN_NORM, true))) { + return 0; + } + if (cgraph->nodes[i]->op == GGML_OP_CLAMP) { + if (!GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_CLAMP, true)) { + return 0; + } + const ggml_op ops[] = { GGML_OP_CLAMP, GGML_OP_MUL_MAT, GGML_OP_CLAMP, GGML_OP_CLAMP, GGML_OP_MUL_MAT, + GGML_OP_CLAMP, GGML_OP_GLU, GGML_OP_CLAMP, GGML_OP_MUL_MAT, GGML_OP_CLAMP }; + const int output_idx = i + 9; + if (!ggml_can_fuse_subgraph(cgraph, i, 10, ops, &output_idx, 1)) { + return 0; + } + ggml_tensor * in0 = cgraph->nodes[i]; + ggml_tensor * mm0 = cgraph->nodes[i + 1]; + ggml_tensor * out0 = cgraph->nodes[i + 2]; + ggml_tensor * in1 = cgraph->nodes[i + 3]; + ggml_tensor * mm1 = cgraph->nodes[i + 4]; + ggml_tensor * out1 = cgraph->nodes[i + 5]; + ggml_tensor * glu = cgraph->nodes[i + 6]; + ggml_tensor * in_down = cgraph->nodes[i + 7]; + ggml_tensor * down = cgraph->nodes[i + 8]; + ggml_tensor * out = cgraph->nodes[output_idx]; + const ggml_glu_op op = ggml_get_glu_op(glu); + if (mm0->src[0]->type != compute_type || mm1->src[0]->type != compute_type || down->src[0]->type != compute_type || + !ggml_cuda_should_fuse_cublas_16bit(ctx, mm0) || !ggml_cuda_should_fuse_cublas_16bit(ctx, mm1) || + !ggml_cuda_should_fuse_cublas_16bit(ctx, down) || in0->src[0] != in1->src[0] || + !ggml_is_contiguous(in0->src[0]) || in0->src[0]->type != GGML_TYPE_F32 || + mm0->src[1] != in0 || out0->src[0] != mm0 || mm1->src[1] != in1 || out1->src[0] != mm1 || + in_down->src[0] != glu || down->src[1] != in_down || out->src[0] != down || + !ggml_are_same_shape(mm0, mm1) || !ggml_are_same_shape(mm0, glu) || + !((glu->src[0] == out0 && glu->src[1] == out1) || (glu->src[0] == out1 && glu->src[1] == out0)) || + (op != GGML_GLU_OP_GEGLU_QUICK && op != GGML_GLU_OP_GEGLU && op != GGML_GLU_OP_SWIGLU)) { + return 0; + } + for (int j : { 0, 2, 3, 5, 7, 9 }) { + const ggml_tensor * clamp = cgraph->nodes[i + j]; + if (clamp->type != GGML_TYPE_F32 || !ggml_is_contiguous(clamp) || !ggml_are_same_shape(clamp, clamp->src[0])) { + return 0; + } + } + const bool shared_input = memcmp(in0->op_params, in1->op_params, 2*sizeof(float)) == 0; + ggml_cuda_pool_alloc input0(ctx.pool(), ggml_nelements(in0)); + ggml_cuda_pool_alloc input1(ctx.pool()); + ggml_cuda_pool_alloc tmp0(ctx.pool(), ggml_nelements(mm0)); + ggml_cuda_pool_alloc tmp1(ctx.pool(), ggml_nelements(mm1)); + ggml_cuda_pool_alloc activated(ctx.pool(), ggml_nelements(glu)); + ggml_cuda_convert_clamp_from_f32((const float *) in0->src[0]->data, input0.get(), ggml_nelements(in0), + ggml_get_op_params_f32(in0, 0), ggml_get_op_params_f32(in0, 1), ctx.stream()); + if (!shared_input) { + input1.alloc(ggml_nelements(in1)); + ggml_cuda_convert_clamp_from_f32((const float *) in1->src[0]->data, input1.get(), ggml_nelements(in1), + ggml_get_op_params_f32(in1, 0), ggml_get_op_params_f32(in1, 1), ctx.stream()); + } + const ggml_cuda_cublas_16bit_fusion f0 = { input0.get(), tmp0.get() }; + const ggml_cuda_cublas_16bit_fusion f1 = { shared_input ? input0.get() : input1.get(), tmp1.get() }; + ggml_cuda_mul_mat_cublas_impl(ctx, mm0->src[0], in0, mm0, nullptr, nullptr, nullptr, &f0); + ggml_cuda_mul_mat_cublas_impl(ctx, mm1->src[0], in1, mm1, nullptr, nullptr, nullptr, &f1); + const bool swapped = glu->src[0] == out1; + const auto bounds = [](const ggml_tensor * clamp) { + return make_float2(ggml_get_op_params_f32(clamp, 0), ggml_get_op_params_f32(clamp, 1)); + }; + ggml_cuda_glu_clamp_to_16bit(swapped ? tmp1.get() : tmp0.get(), swapped ? tmp0.get() : tmp1.get(), + activated.get(), glu->ne[0], ggml_nrows(glu), op, + bounds(swapped ? out1 : out0), bounds(swapped ? out0 : out1), bounds(in_down), ctx.stream()); + const ggml_cuda_cublas_16bit_fusion fd = { activated.get() }; + ggml_cuda_mul_mat_cublas_impl(ctx, down->src[0], in_down, down, out, nullptr, nullptr, &fd); + return 9; + } + const int first = norm_idx >= 0 ? norm_idx : i; + const int prefix = i - first; + const ggml_tensor * norm = prefix ? cgraph->nodes[first] : nullptr; + const ggml_tensor * norm_weight = nullptr; + if (norm) { + const ggml_tensor * mul = cgraph->nodes[first + 1]; + if (prefix != 2 || norm->op != GGML_OP_RMS_NORM || mul->op != GGML_OP_MUL || + (mul->src[0] != norm && mul->src[1] != norm) || cgraph->nodes[i]->src[1] != mul) { + return 0; + } + norm_weight = mul->src[mul->src[0] == norm ? 1 : 0]; + if (norm->type != GGML_TYPE_F32 || !ggml_are_same_shape(norm, mul) || + norm->src[0]->type != GGML_TYPE_F32 || !ggml_is_contiguous(norm->src[0]) || + norm_weight->type != GGML_TYPE_F32 || !ggml_is_contiguous(norm_weight) || + norm_weight->ne[0] != norm->ne[0] || ggml_nrows(norm_weight) != 1) { + return 0; + } + } + if (!norm && GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_FFN_ACT, true)) { + const ggml_op plain_ops[] = { GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_ADD }; + for (int count : { 6, 5, 4 }) { + const int output_idx = i + count - 1; + if (!ggml_can_fuse_subgraph(cgraph, i, count, plain_ops, &output_idx, 1)) { + continue; + } + ggml_tensor * up = cgraph->nodes[i]; + ggml_tensor * bias = cgraph->nodes[i + 1]; + ggml_tensor * gelu = cgraph->nodes[i + 2]; + ggml_tensor * down = cgraph->nodes[i + 3]; + ggml_tensor * out = count >= 5 ? cgraph->nodes[i + 4] : nullptr; + ggml_tensor * residual = count == 6 ? cgraph->nodes[i + 5] : nullptr; + if (up->src[0]->type != compute_type || down->src[0]->type != compute_type || + !ggml_cuda_should_fuse_cublas_16bit(ctx, up) || !ggml_cuda_should_fuse_cublas_16bit(ctx, down) || + !ggml_cuda_can_fuse_cublas_bias(up, bias) || gelu->src[0] != bias || down->src[1] != gelu || + ggml_get_unary_op(gelu) != GGML_UNARY_OP_GELU || + (out && !ggml_cuda_can_fuse_cublas_bias(down, out)) || + (residual && !ggml_cuda_can_fuse_cublas_residual(out, residual))) { + continue; + } + bool internal = false; + if (residual) { + const ggml_tensor * src = residual->src[residual->src[0] == out ? 1 : 0]; + for (int j = i; j < output_idx; ++j) { + internal = internal || src == cgraph->nodes[j] || src->view_src == cgraph->nodes[j]; + } + } + if (internal) { + continue; + } + ggml_cuda_pool_alloc tmp(ctx.pool(), ggml_nelements(up)); + ggml_cuda_pool_alloc activated(ctx.pool(), ggml_nelements(gelu)); + const ggml_cuda_cublas_16bit_fusion fu = { nullptr, tmp.get() }; + ggml_cuda_mul_mat_cublas_impl(ctx, up->src[0], up->src[1], up, nullptr, nullptr, nullptr, &fu); + ggml_cuda_gelu_bias_to_16bit(tmp.get(), (const float *) bias->src[1]->data, + activated.get(), gelu->ne[0], ggml_nrows(gelu), ctx.stream()); + const ggml_cuda_cublas_16bit_fusion fd = { activated.get(), nullptr, out, residual }; + ggml_cuda_mul_mat_cublas_impl(ctx, down->src[0], gelu, down, nullptr, nullptr, nullptr, &fd); + return count - 1; + } + } + const ggml_op ops[] = { GGML_OP_RMS_NORM, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_MUL_MAT, GGML_OP_ADD, + GGML_OP_GLU, GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_ADD }; + for (int count : { 8, 7, 6 }) { + const int output_idx = i + count - 1; + if (!ggml_can_fuse_subgraph(cgraph, first, count + prefix, ops + 2 - prefix, &output_idx, 1)) { + continue; + } + ggml_tensor * mm0 = cgraph->nodes[i]; + ggml_tensor * add0 = cgraph->nodes[i + 1]; + ggml_tensor * mm1 = cgraph->nodes[i + 2]; + ggml_tensor * add1 = cgraph->nodes[i + 3]; + ggml_tensor * glu = cgraph->nodes[i + 4]; + ggml_tensor * down = cgraph->nodes[i + 5]; + ggml_tensor * out = count >= 7 ? cgraph->nodes[i + 6] : nullptr; + ggml_tensor * residual = count == 8 ? cgraph->nodes[i + 7] : nullptr; + const ggml_glu_op op = ggml_get_glu_op(glu); + if (mm0->src[0]->type != compute_type || mm1->src[0]->type != compute_type || down->src[0]->type != compute_type || + !ggml_cuda_should_fuse_cublas_16bit(ctx, mm0) || !ggml_cuda_should_fuse_cublas_16bit(ctx, mm1) || + !ggml_cuda_should_fuse_cublas_16bit(ctx, down) || mm0->src[1] != mm1->src[1] || + !ggml_cuda_can_fuse_cublas_bias(mm0, add0) || !ggml_cuda_can_fuse_cublas_bias(mm1, add1) || + (out && !ggml_cuda_can_fuse_cublas_bias(down, out)) || + (residual && !ggml_cuda_can_fuse_cublas_residual(out, residual)) || + !ggml_are_same_shape(mm0, mm1) || !ggml_are_same_shape(glu, mm0) || + (op != GGML_GLU_OP_GEGLU && op != GGML_GLU_OP_SWIGLU) || down->src[1] != glu || + !((glu->src[0] == add0 && glu->src[1] == add1) || (glu->src[0] == add1 && glu->src[1] == add0))) { + continue; + } + if (residual) { + const ggml_tensor * src = residual->src[residual->src[0] == out ? 1 : 0]; + bool internal = false; + for (int j = first; j < output_idx; ++j) { + internal = internal || src == cgraph->nodes[j] || src->view_src == cgraph->nodes[j]; + } + if (internal) { + continue; + } + } + // Inputs except the final bias and residual are consumed before writing the graph buffer. + ggml_cuda_pool_alloc input(ctx.pool(), ggml_nelements(mm0->src[1])); + ggml_cuda_pool_alloc tmp0(ctx.pool(), ggml_nelements(mm0)); + ggml_cuda_pool_alloc tmp1(ctx.pool(), ggml_nelements(mm1)); + ggml_cuda_pool_alloc activated(ctx.pool(), ggml_nelements(glu)); + if (norm) { + ggml_cuda_rms_norm_mul_to_16bit(norm->src[0], norm_weight, input.get(), ggml_get_op_params_f32(norm, 0), ctx.stream()); + } else { + traits::convert(GGML_TYPE_F32)(mm0->src[1]->data, input.get(), ggml_nelements(mm0->src[1]), ctx.stream()); + } + const ggml_cuda_cublas_16bit_fusion f0 = { input.get(), tmp0.get(), nullptr }; + const ggml_cuda_cublas_16bit_fusion f1 = { input.get(), tmp1.get(), nullptr }; + ggml_cuda_mul_mat_cublas_impl(ctx, mm0->src[0], mm0->src[1], mm0, nullptr, nullptr, nullptr, &f0); + ggml_cuda_mul_mat_cublas_impl(ctx, mm1->src[0], mm1->src[1], mm1, nullptr, nullptr, nullptr, &f1); + const bool swapped = glu->src[0] == add1; + ggml_cuda_glu_bias_to_16bit(swapped ? tmp1.get() : tmp0.get(), swapped ? tmp0.get() : tmp1.get(), + (const float *) (swapped ? add1 : add0)->src[1]->data, + (const float *) (swapped ? add0 : add1)->src[1]->data, + activated.get(), glu->ne[0], ggml_nrows(glu), op, ctx.stream()); + const ggml_cuda_cublas_16bit_fusion fd = { activated.get(), nullptr, out, residual }; + ggml_cuda_mul_mat_cublas_impl(ctx, down->src[0], glu, down, nullptr, nullptr, nullptr, &fd); + return count + prefix - 1; + } + return 0; +} + // try and fuse nodes and return the number of nodes to skip static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph * cgraph, int i) { @@ -4087,6 +4396,96 @@ static int ggml_cuda_try_fuse(ggml_backend_cuda_context * cuda_ctx, ggml_cgraph ggml_tensor * node = cgraph->nodes[i]; + if (node->op == GGML_OP_RMS_NORM && i + 2 < cgraph->n_nodes) { + ggml_tensor * mm = cgraph->nodes[i + 2]; + if (mm->op == GGML_OP_MUL_MAT && ggml_cuda_should_fuse_cublas_16bit(*cuda_ctx, mm)) { + const int skipped = mm->src[0]->type == GGML_TYPE_BF16 ? + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i + 2, i) : + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i + 2, i); + if (skipped > 0) { + return skipped; + } + } + } + + if (node->op == GGML_OP_MUL_MAT && ggml_cuda_should_fuse_cublas_16bit(*cuda_ctx, node)) { + const int skipped = node->src[0]->type == GGML_TYPE_BF16 ? + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i) : + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i); + if (skipped > 0) { + return skipped; + } + const ggml_op ops[] = { GGML_OP_MUL_MAT, GGML_OP_ADD, GGML_OP_ADD }; + for (int count : { 3, 2 }) { + const int output_idx = i + count - 1; + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_BIAS, true) && ggml_can_fuse_subgraph(cgraph, i, count, ops, &output_idx, 1) && + ggml_cuda_can_fuse_cublas_bias(node, cgraph->nodes[i + 1]) && + (count == 2 || ggml_cuda_can_fuse_cublas_residual(cgraph->nodes[i + 1], cgraph->nodes[output_idx])) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, count, &output_idx, 1)) { + const ggml_cuda_cublas_16bit_fusion fusion = { nullptr, nullptr, cgraph->nodes[i + 1], + count == 3 ? cgraph->nodes[output_idx] : nullptr }; + ggml_cuda_mul_mat_cublas_fused(*cuda_ctx, node->src[0], node->src[1], node, nullptr, nullptr, nullptr, &fusion); + return count - 1; + } + } + } + + if (node->op == GGML_OP_GLU && (ggml_get_glu_op(node) == GGML_GLU_OP_GEGLU || ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU) && node->src[1] && + node->type == GGML_TYPE_F32 && node->src[0]->type == GGML_TYPE_F32 && node->src[1]->type == GGML_TYPE_F32 && + ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node->src[1]) && + ggml_cuda_is_aligned(node->src[0], 16) && ggml_cuda_is_aligned(node->src[1], 16) && + ggml_are_same_shape(node, node->src[0]) && ggml_are_same_shape(node, node->src[1])) { + const ggml_op ops[] = { GGML_OP_GLU, GGML_OP_MUL_MAT }; + const int output_idx = i + 1; + if (ggml_can_fuse_subgraph(cgraph, i, 2, ops, &output_idx, 1) && + ggml_cuda_check_fusion_memory_ranges(cgraph, i, 2, &output_idx, 1)) { + ggml_tensor * mm = cgraph->nodes[output_idx]; + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_GLU_16BIT, true) && mm->src[1] == node && ggml_cuda_should_fuse_cublas_16bit(*cuda_ctx, mm)) { + ggml_cuda_mul_mat_cublas_fused(*cuda_ctx, mm->src[0], node, mm, nullptr, nullptr, node); + return 1; + } + } + } + + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_CLAMP, true) && node->op == GGML_OP_CLAMP && i + 1 < cgraph->n_nodes && + node->type == GGML_TYPE_F32 && ggml_is_contiguous(node->src[0]) && ggml_is_contiguous(node)) { + ggml_tensor * mm = cgraph->nodes[i + 1]; + if (mm->op == GGML_OP_MUL_MAT && mm->src[1] == node && ggml_cuda_should_fuse_cublas_16bit(*cuda_ctx, mm)) { + const int skipped = mm->src[0]->type == GGML_TYPE_BF16 ? + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i) : + ggml_cuda_try_fuse_16bit_ffn(*cuda_ctx, cgraph, i); + if (skipped > 0) { + return skipped; + } + const ggml_op ops[] = { GGML_OP_CLAMP, GGML_OP_MUL_MAT, GGML_OP_CLAMP }; + for (int count : { 3, 2 }) { + const int output_idx = i + count - 1; + if (!ggml_can_fuse_subgraph(cgraph, i, count, ops, &output_idx, 1)) { + continue; + } + ggml_tensor * out = cgraph->nodes[output_idx]; + if (count == 3 && (out->src[0] != mm || out->type != GGML_TYPE_F32 || !ggml_is_contiguous(out))) { + continue; + } + // BLAS consumes both inputs before the final conversion writes the output. + ggml_cuda_mul_mat_cublas_fused(*cuda_ctx, mm->src[0], node, mm, count == 3 ? out : nullptr, node); + return count - 1; + } + } + } + + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_FUSE_CUBLAS_CLAMP, true) && node->op == GGML_OP_MUL_MAT && ggml_can_fuse(cgraph, i, { GGML_OP_MUL_MAT, GGML_OP_CLAMP }) && + ggml_cuda_should_fuse_cublas_16bit(*cuda_ctx, node)) { + ggml_tensor * clamp = cgraph->nodes[i + 1]; + if (clamp->src[0] == node && clamp->type == GGML_TYPE_F32 && ggml_is_contiguous(clamp)) { + ggml_cuda_mul_mat_cublas_fused(*cuda_ctx, node->src[0], node->src[1], node, clamp); + return 1; + } + } + + + + if (node->op == GGML_OP_MUL_MAT_ID && cuda_ctx->stream_context().concurrent_events.empty() && ggml_cuda_match_shared_expert(cgraph, i, i + 3) && ggml_cuda_should_fuse_mul_mat_vec_q(cgraph->nodes[i + 2]->src[1])) { diff --git a/ggml/src/ggml-cuda/norm.cu b/ggml/src/ggml-cuda/norm.cu index 3f0ea6f2a474..250ac4e8abc8 100644 --- a/ggml/src/ggml-cuda/norm.cu +++ b/ggml/src/ggml-cuda/norm.cu @@ -1,4 +1,5 @@ #include "norm.cuh" +#include "convert.cuh" #include template @@ -81,9 +82,9 @@ static __global__ void group_norm_f32(const float * x, float * dst, const int gr } } -template +template static __global__ void rms_norm_f32(const float * x, - float * dst, + dst_t * dst, const int ncols, const int nchannels, const int nsamples, @@ -124,7 +125,7 @@ static __global__ void rms_norm_f32(const float * x, for (int sample = blockIdx.z; sample < nsamples; sample += gridDim.z) { for (int channel = blockIdx.y; channel < nchannels; channel += gridDim.y) { const float * xc = x + sample*stride_sample + channel*stride_channel + row*stride_row; - float * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; + dst_t * dstc = dst + ((sample*nchannels + channel)*nrows + row)*ncols; [[maybe_unused]] const float * mulc = nullptr; if constexpr (do_multiply) { @@ -147,7 +148,7 @@ static __global__ void rms_norm_f32(const float * x, const float4 * x4 = (const float4 *) xc; const float4 * mul4 = (const float4 *) mulc; const float4 * add4 = (const float4 *) addc; - float4 * dst4 = (float4 *) dstc; + [[maybe_unused]] float4 * dst4 = (float4 *) dstc; float4 xs[vpt]; float tmp = 0.0f; @@ -188,7 +189,19 @@ static __global__ void rms_norm_f32(const float * x, r.z *= scale_out; r.w *= scale_out; } - dst4[i] = r; + if constexpr (std::is_same_v) { + dst4[i] = r; + } else { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(r.x), "+v"(r.y), "+v"(r.z), "+v"(r.w)); +#else + asm volatile("" : "+f"(r.x), "+f"(r.y), "+f"(r.z), "+f"(r.w)); +#endif + dstc[4*i + 0] = ggml_cuda_cast(r.x); + dstc[4*i + 1] = ggml_cuda_cast(r.y); + dstc[4*i + 2] = ggml_cuda_cast(r.z); + dstc[4*i + 3] = ggml_cuda_cast(r.w); + } } } } else { @@ -212,7 +225,15 @@ static __global__ void rms_norm_f32(const float * x, dstc[col] = scale * xc[col] * mulc[mul_col] + addc[add_col]; } else if constexpr (do_multiply) { const int mul_col = fastmodulo(col, mul_ncols_packed); - dstc[col] = scale * xc[col] * mulc[mul_col]; + float value = scale * xc[col] * mulc[mul_col]; + if constexpr (!std::is_same_v) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(value)); +#else + asm volatile("" : "+f"(value)); +#endif + } + dstc[col] = ggml_cuda_cast(value); } else if constexpr (do_scale) { dstc[col] = scale_out * (scale * xc[col]); } else { @@ -387,7 +408,7 @@ static void group_norm_f32_cuda( do { \ const dim3 block_dims(BLOCK_SIZE, 1, 1); \ const ggml_cuda_kernel_launch_params launch_params{ blocks_num, block_dims, block_dims.x > WARP_SIZE ? 32 * sizeof(float) : 0, stream }; \ - ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ + ggml_cuda_kernel_launch(rms_norm_f32, launch_params, \ x, dst, ncols, nchannels, nsamples, stride_row, stride_channel, stride_sample, eps, \ mul, mul_stride_row, mul_stride_channel, mul_stride_sample, \ mul_ncols_packed, mul_nrows_packed, mul_nchannels_packed, mul_nsamples_packed, \ @@ -395,10 +416,10 @@ static void group_norm_f32_cuda( add_ncols_packed, add_nrows_packed, add_nchannels_packed, add_nsamples_packed, scale_out); \ } while (0) -template +template static void rms_norm_f32_dispatch( const dim3 blocks_num, - const ggml_tensor * x_t, float * dst, const int ncols, const int nchannels, const int nsamples, + const ggml_tensor * x_t, dst_t * dst, const int ncols, const int nchannels, const int nsamples, const int64_t stride_row, const int64_t stride_channel, const int64_t stride_sample, const float eps, const ggml_tensor * mul_t, const int64_t mul_stride_row, const int64_t mul_stride_channel, const int64_t mul_stride_sample, const uint3 mul_ncols_packed, const uint3 mul_nrows_packed, const uint3 mul_nchannels_packed, const uint3 mul_nsamples_packed, @@ -449,6 +470,19 @@ static void rms_norm_f32_dispatch( } #undef RMS_NORM_LAUNCH +template +void ggml_cuda_rms_norm_mul_to_16bit(const ggml_tensor * x, const ggml_tensor * weight, T * dst, float eps, cudaStream_t stream) { + const uint3 one = init_fastdiv_values(1); + const uint3 zero = make_uint3(0, 0, 0); + rms_norm_f32_dispatch( + dim3(x->ne[1]), x, dst, x->ne[0], 1, 1, x->nb[1]/sizeof(float), 0, 0, eps, + weight, 0, 0, 0, init_fastdiv_values(x->ne[0]), one, one, one, + nullptr, 0, 0, 0, zero, zero, zero, zero, stream); +} + +template void ggml_cuda_rms_norm_mul_to_16bit(const ggml_tensor *, const ggml_tensor *, half *, float, cudaStream_t); +template void ggml_cuda_rms_norm_mul_to_16bit(const ggml_tensor *, const ggml_tensor *, nv_bfloat16 *, float, cudaStream_t); + template static void rms_norm_f32_cuda( const ggml_tensor * x_t, float * dst, const int ncols, const int nrows, const int nchannels, const int nsamples, diff --git a/ggml/src/ggml-cuda/norm.cuh b/ggml/src/ggml-cuda/norm.cuh index 95618df5fd4d..7959ebd12ad7 100644 --- a/ggml/src/ggml-cuda/norm.cuh +++ b/ggml/src/ggml-cuda/norm.cuh @@ -18,3 +18,6 @@ void ggml_cuda_op_rms_norm_fused_add(ggml_backend_cuda_context & ctx, void ggml_cuda_op_rms_norm_back(ggml_backend_cuda_context & ctx, ggml_tensor * dst); void ggml_cuda_op_l2_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst); + +template +void ggml_cuda_rms_norm_mul_to_16bit(const ggml_tensor * x, const ggml_tensor * weight, T * dst, float eps, cudaStream_t stream); diff --git a/ggml/src/ggml-cuda/unary.cu b/ggml/src/ggml-cuda/unary.cu index b3b6aa48d3ac..58581cf9c479 100644 --- a/ggml/src/ggml-cuda/unary.cu +++ b/ggml/src/ggml-cuda/unary.cu @@ -303,8 +303,8 @@ void ggml_cuda_op_softplus(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { } /* gated ops */ -template -static __global__ void unary_gated_op_kernel(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1) { +template +static __global__ void unary_gated_op_kernel(const T * x, const T * g, dst_t * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1) { ggml_cuda_pdl_lc(); const int64_t i = int64_t(blockDim.x)*blockIdx.x + threadIdx.x; @@ -317,14 +317,22 @@ static __global__ void unary_gated_op_kernel(const T * x, const T * g, T * dst, const int64_t j1 = o0 == o1 ? j0 : (i / n) * o1 + (i % n); ggml_cuda_pdl_sync(); - dst[i] = ggml_cuda_cast(op(ggml_cuda_cast(x[j0])) * ggml_cuda_cast(g[j1])); + float value = op(ggml_cuda_cast(x[j0])) * ggml_cuda_cast(g[j1]); + if constexpr (!std::is_same_v) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(value)); +#else + asm volatile("" : "+f"(value)); +#endif + } + dst[i] = ggml_cuda_cast(value); } -template -static void unary_gated_cuda(const T * x, const T * g, T * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, cudaStream_t stream) { +template +static void unary_gated_cuda(const T * x, const T * g, dst_t * dst, const int64_t k, const int64_t n, const int64_t o0, const int64_t o1, cudaStream_t stream) { const int64_t num_blocks = (k + CUDA_GLU_BLOCK_SIZE - 1) / CUDA_GLU_BLOCK_SIZE; const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params((dim3)num_blocks, CUDA_GLU_BLOCK_SIZE, 0, stream); - ggml_cuda_kernel_launch(unary_gated_op_kernel, launch_params, x, g, dst, k, n, o0, o1); + ggml_cuda_kernel_launch(unary_gated_op_kernel, launch_params, x, g, dst, k, n, o0, o1); } template @@ -394,6 +402,166 @@ void ggml_cuda_op_reglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_op_unary_gated(ctx, dst); } +template +void ggml_cuda_glu_to_16bit(const float * x, const float * gate, T * dst, int64_t n, ggml_glu_op op, cudaStream_t stream) { + switch (op) { + case GGML_GLU_OP_GEGLU: + unary_gated_cuda(x, gate, dst, n, n, n, n, stream); + break; + case GGML_GLU_OP_SWIGLU: + unary_gated_cuda(x, gate, dst, n, n, n, n, stream); + break; + default: + GGML_ABORT("unsupported GLU conversion operation"); + } +} + +template +static __global__ void glu_bias_to_16bit(const T * x, const T * gate, const float * x_bias, const float * gate_bias, + T * dst, int64_t ncols, int64_t nrows, float2 x_bounds = {}, float2 gate_bounds = {}, float2 dst_bounds = {}) { + const int64_t col = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (col >= ncols) { + return; + } + const float bx = clamped ? 0.0f : x_bias[col]; + const float bg = gated && !clamped ? gate_bias[col] : 0.0f; + for (int64_t row = blockIdx.y; row < nrows; row += gridDim.y) { + const int64_t i = row*ncols + col; + float a = ggml_cuda_cast(x[i]); + float b = gated ? ggml_cuda_cast(gate[i]) : 1.0f; + if constexpr (clamped) { +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(a), "+v"(b)); +#else + asm volatile("" : "+f"(a), "+f"(b)); +#endif + a = fminf(fmaxf(a, x_bounds.x), x_bounds.y); + b = fminf(fmaxf(b, gate_bounds.x), gate_bounds.y); + } else { + a += bx; + if constexpr (gated) { + b += bg; + } + } +#if defined(GGML_USE_HIP) + // Retain the FP32 bias rounding before the activation. + asm volatile("" : "+v"(a), "+v"(b)); +#else + asm volatile("" : "+f"(a), "+f"(b)); +#endif + float v = op(a)*b; +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(v)); +#else + asm volatile("" : "+f"(v)); +#endif + if constexpr (clamped) { + v = fminf(fmaxf(v, dst_bounds.x), dst_bounds.y); + } + dst[i] = ggml_cuda_cast(v); + } +} + +template +void ggml_cuda_glu_clamp_to_16bit(const T * x, const T * gate, T * dst, int64_t ncols, int64_t nrows, + ggml_glu_op op, float2 x_bounds, float2 gate_bounds, float2 dst_bounds, cudaStream_t stream) { + const dim3 blocks((ncols + CUDA_GLU_BLOCK_SIZE - 1)/CUDA_GLU_BLOCK_SIZE, std::min(nrows, int64_t(65535))); + switch (op) { + case GGML_GLU_OP_GEGLU_QUICK: + glu_bias_to_16bit<<>>(x, gate, nullptr, nullptr, dst, ncols, nrows, x_bounds, gate_bounds, dst_bounds); + break; + case GGML_GLU_OP_GEGLU: + glu_bias_to_16bit<<>>(x, gate, nullptr, nullptr, dst, ncols, nrows, x_bounds, gate_bounds, dst_bounds); + break; + case GGML_GLU_OP_SWIGLU: + glu_bias_to_16bit<<>>(x, gate, nullptr, nullptr, dst, ncols, nrows, x_bounds, gate_bounds, dst_bounds); + break; + default: + GGML_ABORT("unsupported GLU clamp operation"); + } +} + +template +void ggml_cuda_glu_bias_to_16bit(const T * x, const T * gate, const float * x_bias, const float * gate_bias, + T * dst, int64_t ncols, int64_t nrows, ggml_glu_op op, cudaStream_t stream) { + const dim3 blocks((ncols + CUDA_GLU_BLOCK_SIZE - 1)/CUDA_GLU_BLOCK_SIZE, std::min(nrows, int64_t(65535))); + switch (op) { + case GGML_GLU_OP_GEGLU: + glu_bias_to_16bit<<>>(x, gate, x_bias, gate_bias, dst, ncols, nrows); + break; + case GGML_GLU_OP_SWIGLU: + glu_bias_to_16bit<<>>(x, gate, x_bias, gate_bias, dst, ncols, nrows); + break; + default: + GGML_ABORT("unsupported GLU bias operation"); + } +} + + +template +struct alignas(2*sizeof(T)) glu_pair { + T v[2]; +}; + +template +static __global__ void gelu_bias_to_16bit_vec2(const glu_pair * x, const float2 * bias, glu_pair * dst, + int64_t ncols, int64_t nrows) { + const int64_t col = int64_t(blockIdx.x)*blockDim.x + threadIdx.x; + if (col >= ncols) { + return; + } + const float2 b = bias[col]; + for (int64_t row = blockIdx.y; row < nrows; row += gridDim.y) { + const int64_t i = row*ncols + col; + const auto in = x[i]; + float a0 = ggml_cuda_cast(in.v[0]) + b.x; + float a1 = ggml_cuda_cast(in.v[1]) + b.y; +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(a0), "+v"(a1)); +#else + asm volatile("" : "+f"(a0), "+f"(a1)); +#endif + float v0 = op_gelu(a0); + float v1 = op_gelu(a1); +#if defined(GGML_USE_HIP) + asm volatile("" : "+v"(v0), "+v"(v1)); +#else + asm volatile("" : "+f"(v0), "+f"(v1)); +#endif + dst[i] = {{ggml_cuda_cast(v0), ggml_cuda_cast(v1)}}; + } +} + +template +void ggml_cuda_gelu_bias_to_16bit(const T * x, const float * bias, T * dst, int64_t ncols, int64_t nrows, cudaStream_t stream) { +#if defined(GGML_USE_HIP) + const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc; + if (GGML_CUDA_OPT_ENABLED(GGML_CUDA_GELU_BIAS_VEC, true) && GGML_CUDA_CC_IS_RDNA3_5(cc) && ncols % 2 == 0 && + reinterpret_cast(x) % alignof(glu_pair) == 0 && + reinterpret_cast(bias) % alignof(float2) == 0 && + reinterpret_cast(dst) % alignof(glu_pair) == 0) { + const dim3 blocks((ncols/2 + CUDA_GLU_BLOCK_SIZE - 1)/CUDA_GLU_BLOCK_SIZE, std::min(nrows, int64_t(65535))); + gelu_bias_to_16bit_vec2<<>>( + reinterpret_cast *>(x), reinterpret_cast(bias), + reinterpret_cast *>(dst), ncols/2, nrows); + return; + } +#endif + const dim3 blocks((ncols + CUDA_GLU_BLOCK_SIZE - 1)/CUDA_GLU_BLOCK_SIZE, std::min(nrows, int64_t(65535))); + glu_bias_to_16bit<<>>(x, nullptr, bias, nullptr, dst, ncols, nrows); +} + +template void ggml_cuda_glu_clamp_to_16bit(const half *, const half *, half *, int64_t, int64_t, ggml_glu_op, float2, float2, float2, cudaStream_t); +template void ggml_cuda_glu_clamp_to_16bit(const nv_bfloat16 *, const nv_bfloat16 *, nv_bfloat16 *, int64_t, int64_t, ggml_glu_op, float2, float2, float2, cudaStream_t); + +template void ggml_cuda_gelu_bias_to_16bit(const half *, const float *, half *, int64_t, int64_t, cudaStream_t); +template void ggml_cuda_gelu_bias_to_16bit(const nv_bfloat16 *, const float *, nv_bfloat16 *, int64_t, int64_t, cudaStream_t); + +template void ggml_cuda_glu_to_16bit(const float *, const float *, half *, int64_t, ggml_glu_op, cudaStream_t); +template void ggml_cuda_glu_bias_to_16bit(const half *, const half *, const float *, const float *, half *, int64_t, int64_t, ggml_glu_op, cudaStream_t); +template void ggml_cuda_glu_to_16bit(const float *, const float *, nv_bfloat16 *, int64_t, ggml_glu_op, cudaStream_t); +template void ggml_cuda_glu_bias_to_16bit(const nv_bfloat16 *, const nv_bfloat16 *, const float *, const float *, nv_bfloat16 *, int64_t, int64_t, ggml_glu_op, cudaStream_t); + void ggml_cuda_op_geglu(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { ggml_cuda_op_unary_gated(ctx, dst); } diff --git a/ggml/src/ggml-cuda/unary.cuh b/ggml/src/ggml-cuda/unary.cuh index c6ffdb9d63e7..9a93fde57b24 100644 --- a/ggml/src/ggml-cuda/unary.cuh +++ b/ggml/src/ggml-cuda/unary.cuh @@ -136,3 +136,16 @@ __device__ __forceinline__ float ggml_cuda_op_glu_single(ggml_glu_op op, float g return up * gate; } } + +template +void ggml_cuda_glu_to_16bit(const float * x, const float * gate, T * dst, int64_t n, ggml_glu_op op, cudaStream_t stream); +template +void ggml_cuda_glu_bias_to_16bit(const T * x, const T * gate, const float * x_bias, const float * gate_bias, + T * dst, int64_t ncols, int64_t nrows, ggml_glu_op op, cudaStream_t stream); + +template +void ggml_cuda_gelu_bias_to_16bit(const T * x, const float * bias, T * dst, int64_t ncols, int64_t nrows, cudaStream_t stream); + +template +void ggml_cuda_glu_clamp_to_16bit(const T * x, const T * gate, T * dst, int64_t ncols, int64_t nrows, + ggml_glu_op op, float2 x_bounds, float2 gate_bounds, float2 dst_bounds, cudaStream_t stream); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3e52e4abe10c..f4187ecd121c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -5574,6 +5574,166 @@ struct test_mul_mat : public test_case { } }; +struct test_glu_mat : public test_mul_mat { + const bool extra_consumer; + const bool force_f32; + const ggml_glu_op glu_op; + test_glu_mat(ggml_type type, int64_t n, int64_t k = 512, bool extra_consumer = false, bool force_f32 = false, ggml_glu_op glu_op = GGML_GLU_OP_GEGLU) + : test_mul_mat(type, GGML_TYPE_F32, 256, n, k, {1, 1}, {1, 1}), extra_consumer(extra_consumer), force_f32(force_f32), glu_op(glu_op) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return glu_op == GGML_GLU_OP_GEGLU ? "GEGLU_MAT" : "SWIGLU_MAT"; } + std::string vars() override { return test_mul_mat::vars() + "," + VARS_TO_STR2(extra_consumer, force_f32); } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * mm = test_mul_mat::build_graph(ctx); + ggml_tensor * gate = ggml_dup_tensor(ctx, mm->src[1]); + mm->src[1] = glu_op == GGML_GLU_OP_GEGLU ? ggml_geglu_split(ctx, mm->src[1], gate) : ggml_swiglu_split(ctx, mm->src[1], gate); + if (force_f32) { + ggml_prec_set_acc(mm, GGML_PREC_F32); + } + return extra_consumer ? ggml_add(ctx, mm, ggml_mul_mat(ctx, mm->src[0], mm->src[1])) : mm; + } +}; + +struct test_mul_mat_clamp : public test_mul_mat { + bool input_clamp; + bool extra_consumer; + bool force_f32; + + test_mul_mat_clamp(ggml_type type, int64_t m, int64_t n, int64_t k, bool input_clamp = false, bool extra_consumer = false, + bool force_f32 = false, int64_t k_v = 0) + : test_mul_mat(type, GGML_TYPE_F32, m, n, k, {1, 1}, {1, 1}, {0, 1, 2, 3}, k_v), + input_clamp(input_clamp), extra_consumer(extra_consumer), force_f32(force_f32) {} + + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return "MUL_MAT_CLAMP"; } + std::string vars() override { + return test_mul_mat::vars() + "," + VARS_TO_STR3(input_clamp, extra_consumer, force_f32); + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * mm = test_mul_mat::build_graph(ctx); + if (input_clamp) { + ggml_tensor * x = ggml_is_contiguous(mm->src[1]) ? mm->src[1] : ggml_cont(ctx, mm->src[1]); + mm->src[1] = ggml_clamp(ctx, x, -0.12345f, 0.45678f); + } + if (force_f32) { + ggml_prec_set_acc(mm, GGML_PREC_F32); + } + ggml_tensor * out = ggml_clamp(ctx, mm, -0.34567f, 0.23456f); + return extra_consumer ? ggml_add(ctx, out, mm) : out; + } +}; + +struct test_mul_mat_bias : public test_mul_mat { + const bool extra_consumer, force_f32, residual; + test_mul_mat_bias(ggml_type type, int64_t m, int64_t n, bool extra_consumer = false, bool force_f32 = false, int64_t k_v = 0, bool residual = false) + : test_mul_mat(type, GGML_TYPE_F32, m, n, 512, {1, 1}, {1, 1}, {0, 1, 2, 3}, k_v), + extra_consumer(extra_consumer), force_f32(force_f32), residual(residual) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return "MUL_MAT_BIAS"; } + std::string vars() override { return test_mul_mat::vars() + "," + VARS_TO_STR3(extra_consumer, force_f32, residual); } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * mm = test_mul_mat::build_graph(ctx); + if (force_f32) { + ggml_prec_set_acc(mm, GGML_PREC_F32); + } + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, mm->ne[0]); + ggml_tensor * out = ggml_add(ctx, mm, bias); + if (residual) { + out = ggml_add(ctx, out, ggml_dup_tensor(ctx, mm)); + } + return extra_consumer ? ggml_add(ctx, out, mm) : out; + } +}; + +struct test_16bit_ffn_bias : public test_case { + const ggml_type type; + const bool mixed_weights; + const int64_t n, hidden, width; + const ggml_glu_op op; + const int extra_consumer; + const bool output_bias, force_f32, swapped, residual, normalized, plain; + test_16bit_ffn_bias(ggml_type type, int64_t n, int64_t hidden = 768, ggml_glu_op op = GGML_GLU_OP_SWIGLU, + int extra_consumer = 0, bool output_bias = true, bool force_f32 = false, bool swapped = false, bool mixed_weights = false, bool residual = false, bool normalized = false, bool plain = false, int64_t width = 512) + : type(type), mixed_weights(mixed_weights), n(n), hidden(hidden), width(width), op(op), extra_consumer(extra_consumer), output_bias(output_bias), force_f32(force_f32), swapped(swapped), residual(residual), normalized(normalized), plain(plain) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return type == GGML_TYPE_BF16 ? "BF16_FFN_BIAS" : "F16_FFN_BIAS"; } + std::string vars() override { + return VARS_TO_STR2(type, mixed_weights) + "," + VARS_TO_STR4(n, hidden, width, extra_consumer) + "," + VARS_TO_STR5(output_bias, force_f32, swapped, residual, normalized) + "," + VAR_TO_STR(plain) + ",op=" + std::to_string(op); + } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, n); + x = ggml_scale(ctx, x, 0.125f); + ggml_tensor * residual_input = x; + if (normalized) { + x = ggml_mul(ctx, ggml_rms_norm(ctx, x, 1e-6f), ggml_new_tensor_1d(ctx, GGML_TYPE_F32, width)); + } + ggml_tensor * w0 = ggml_new_tensor_2d(ctx, type, width, hidden); + ggml_type other = type == GGML_TYPE_F16 ? GGML_TYPE_BF16 : GGML_TYPE_F16; + ggml_tensor * w1 = ggml_new_tensor_2d(ctx, mixed_weights ? other : type, width, hidden); + ggml_tensor * b0 = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, hidden); + ggml_tensor * b1 = ggml_dup_tensor(ctx, b0); + ggml_tensor * mm0 = ggml_mul_mat(ctx, w0, x); + ggml_tensor * mm1 = ggml_mul_mat(ctx, w1, x); + if (force_f32) { + ggml_prec_set_acc(mm0, GGML_PREC_F32); + } + ggml_tensor * a0 = ggml_add(ctx, mm0, b0); + ggml_tensor * a1 = ggml_add(ctx, mm1, b1); + ggml_tensor * first = swapped ? a1 : a0; + ggml_tensor * second = swapped ? a0 : a1; + ggml_tensor * glu = plain ? ggml_gelu(ctx, a0) : op == GGML_GLU_OP_SWIGLU ? ggml_swiglu_split(ctx, first, second) : ggml_geglu_split(ctx, first, second); + ggml_tensor * down = ggml_new_tensor_2d(ctx, type, hidden, width); + ggml_tensor * out = ggml_mul_mat(ctx, down, glu); + if (output_bias) { + out = ggml_add(ctx, out, ggml_new_tensor_1d(ctx, GGML_TYPE_F32, width)); + } + if (residual) { + out = ggml_add(ctx, out, residual_input); + } + if (extra_consumer) { + ggml_tensor * extra = extra_consumer == 1 ? a0 : glu; + out = ggml_add(ctx, out, ggml_mul_mat(ctx, down, extra)); + } + return out; + } +}; + +struct test_16bit_ffn_clamp : public test_case { + ggml_type type; + int64_t n; + int extra; + bool different_bounds, force_f32, swapped; + test_16bit_ffn_clamp(ggml_type type, int64_t n, int extra = 0, bool different_bounds = false, bool force_f32 = false, bool swapped = false) + : type(type), n(n), extra(extra), different_bounds(different_bounds), force_f32(force_f32), swapped(swapped) {} + bool run_whole_graph() override { return true; } + std::string op_desc(ggml_tensor *) override { return type == GGML_TYPE_F16 ? "F16_FFN_CLAMP" : "BF16_FFN_CLAMP"; } + std::string vars() override { return VARS_TO_STR6(type, n, extra, different_bounds, force_f32, swapped); } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t width = 512, hidden = 767; + ggml_tensor * x = ggml_scale(ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, width, n), 0.125f); + ggml_tensor * c0 = ggml_clamp(ctx, x, -1.0f, 1.0f); + ggml_tensor * c1 = ggml_clamp(ctx, x, different_bounds ? -0.8f : -1.0f, different_bounds ? 0.7f : 1.0f); + ggml_tensor * m0 = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx, type, width, hidden), c0); + ggml_tensor * m1 = ggml_mul_mat(ctx, ggml_new_tensor_2d(ctx, type, width, hidden), c1); + if (force_f32) { + ggml_prec_set_acc(m0, GGML_PREC_F32); + } + ggml_tensor * a0 = ggml_clamp(ctx, m0, -2.0f, 2.0f); + ggml_tensor * a1 = ggml_clamp(ctx, m1, -3.0f, 3.0f); + ggml_tensor * glu = ggml_geglu_quick_split(ctx, swapped ? a1 : a0, swapped ? a0 : a1); + ggml_tensor * input = ggml_clamp(ctx, glu, -1.5f, 1.5f); + ggml_tensor * w = ggml_new_tensor_2d(ctx, type, hidden, width); + ggml_tensor * out = ggml_clamp(ctx, ggml_mul_mat(ctx, w, input), -4.0f, 4.0f); + if (extra) { + out = ggml_add(ctx, out, ggml_mul_mat(ctx, w, extra == 1 ? a0 : glu)); + } + return out; + } +}; + // GGML_HINT_SRC0_IS_HADAMARD struct test_mul_mat_hadamard : public test_mul_mat { test_mul_mat_hadamard(ggml_type type_a = GGML_TYPE_F32, ggml_type type_b = GGML_TYPE_F32, @@ -10339,6 +10499,64 @@ static std::vector> make_test_cases_eval() { std::vector> test_cases; std::default_random_engine rng(0); + for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_BF16 }) { + for (int64_t n : { 1, 31, 32, 64, 257 }) { + test_cases.emplace_back(new test_mul_mat_bias(type, 256, n)); + test_cases.emplace_back(new test_mul_mat_bias(type, 255, n)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, n)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, n, 767, GGML_GLU_OP_GEGLU, 0, true, false, false, false, true, false, true)); + } + for (int64_t n : { 1, 31, 32, 65, 257 }) { + test_cases.emplace_back(new test_16bit_ffn_clamp(type, n)); + test_cases.emplace_back(new test_16bit_ffn_clamp(type, n, 0, true)); + } + test_cases.emplace_back(new test_16bit_ffn_clamp(type, 65, 1)); + test_cases.emplace_back(new test_16bit_ffn_clamp(type, 65, 2)); + test_cases.emplace_back(new test_16bit_ffn_clamp(type, 65, 0, false, true)); + test_cases.emplace_back(new test_16bit_ffn_clamp(type, 65, 0, false, false, true)); + for (int extra : { 0, 1, 2 }) { + test_cases.emplace_back(new test_16bit_ffn_bias(type, 65, 768, GGML_GLU_OP_GEGLU, extra, true, false, false, false, true, false, true)); + } + test_cases.emplace_back(new test_16bit_ffn_bias(type, 65, 768, GGML_GLU_OP_GEGLU, 0, false, false, false, false, false, false, true)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 65, 768, GGML_GLU_OP_GEGLU, 0, true, true, false, false, true, false, true)); + test_cases.emplace_back(new test_mul_mat_bias(type, 256, 64, true)); + test_cases.emplace_back(new test_mul_mat_bias(type, 256, 64, false, true)); + test_cases.emplace_back(new test_mul_mat_bias(type, 256, 64, false, false, 576)); + for (int64_t rows : {31, 32, 65}) { + test_cases.emplace_back(new test_mul_mat_bias(type, 255, rows, false, false, 0, true)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, rows, 768, GGML_GLU_OP_SWIGLU, 0, true, false, false, false, true)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, rows, 768, GGML_GLU_OP_SWIGLU, 0, true, false, false, false, true, true)); + } + test_cases.emplace_back(new test_mul_mat_bias(type, 256, 64, true, false, 0, true)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 65, 768, GGML_GLU_OP_GEGLU, 1, true, false, true, false, true)); + for (ggml_glu_op op : { GGML_GLU_OP_SWIGLU, GGML_GLU_OP_GEGLU }) { + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 767, op)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, op, 0, false)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, op, 1)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, op, 2)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, op, 0, true, true)); + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, op, 0, true, false, true)); + } + test_cases.emplace_back(new test_16bit_ffn_bias(type, 64, 768, GGML_GLU_OP_SWIGLU, 0, true, false, false, true)); + } + + for (ggml_type type : { GGML_TYPE_F16, GGML_TYPE_BF16 }) { + for (int64_t n : { 31, 32, 65 }) { + test_cases.emplace_back(new test_glu_mat(type, n)); + test_cases.emplace_back(new test_glu_mat(type, n, 512, false, false, GGML_GLU_OP_SWIGLU)); + test_cases.emplace_back(new test_mul_mat_clamp(type, 255, n, 512, true)); + } + for (int64_t width : { 511, 1152, 1280 }) { + test_cases.emplace_back(new test_16bit_ffn_bias(type, 65, 768, GGML_GLU_OP_SWIGLU, 0, true, false, false, false, true, true, false, width)); + } + test_cases.emplace_back(new test_glu_mat(type, 65, 512, true)); + test_cases.emplace_back(new test_glu_mat(type, 65, 512, false, true)); + test_cases.emplace_back(new test_mul_mat_clamp(type, 256, 65, 512, false)); + test_cases.emplace_back(new test_mul_mat_clamp(type, 256, 65, 512, true, true)); + test_cases.emplace_back(new test_mul_mat_clamp(type, 256, 65, 512, true, false, true)); + } + + // unary ops // TODO: add GGML_UNARY_OP_SOFTPLUS test that is sensitive to using `log1p(exp(x))` vs `log(exp(x) + 1)` for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32, GGML_TYPE_BF16}) {